{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTMs for Text Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import collections\n",
    "import math\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from matplotlib import pylab\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "import tensorflow as tf\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading Stories\n",
    "Stories are automatically downloaded from https://www.cs.cmu.edu/~spok/grimmtmp/, if not detected in the disk. The total size of stories is around ~500KB. The dataset consists of 100 stories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading file:  stories\\001.txt\n",
      "File  001.txt  already exists.\n",
      "Downloading file:  stories\\002.txt\n",
      "File  002.txt  already exists.\n",
      "Downloading file:  stories\\003.txt\n",
      "File  003.txt  already exists.\n",
      "Downloading file:  stories\\004.txt\n",
      "File  004.txt  already exists.\n",
      "Downloading file:  stories\\005.txt\n",
      "File  005.txt  already exists.\n",
      "Downloading file:  stories\\006.txt\n",
      "File  006.txt  already exists.\n",
      "Downloading file:  stories\\007.txt\n",
      "File  007.txt  already exists.\n",
      "Downloading file:  stories\\008.txt\n",
      "File  008.txt  already exists.\n",
      "Downloading file:  stories\\009.txt\n",
      "File  009.txt  already exists.\n",
      "Downloading file:  stories\\010.txt\n",
      "File  010.txt  already exists.\n",
      "Downloading file:  stories\\011.txt\n",
      "File  011.txt  already exists.\n",
      "Downloading file:  stories\\012.txt\n",
      "File  012.txt  already exists.\n",
      "Downloading file:  stories\\013.txt\n",
      "File  013.txt  already exists.\n",
      "Downloading file:  stories\\014.txt\n",
      "File  014.txt  already exists.\n",
      "Downloading file:  stories\\015.txt\n",
      "File  015.txt  already exists.\n",
      "Downloading file:  stories\\016.txt\n",
      "File  016.txt  already exists.\n",
      "Downloading file:  stories\\017.txt\n",
      "File  017.txt  already exists.\n",
      "Downloading file:  stories\\018.txt\n",
      "File  018.txt  already exists.\n",
      "Downloading file:  stories\\019.txt\n",
      "File  019.txt  already exists.\n",
      "Downloading file:  stories\\020.txt\n",
      "File  020.txt  already exists.\n",
      "Downloading file:  stories\\021.txt\n",
      "File  021.txt  already exists.\n",
      "Downloading file:  stories\\022.txt\n",
      "File  022.txt  already exists.\n",
      "Downloading file:  stories\\023.txt\n",
      "File  023.txt  already exists.\n",
      "Downloading file:  stories\\024.txt\n",
      "File  024.txt  already exists.\n",
      "Downloading file:  stories\\025.txt\n",
      "File  025.txt  already exists.\n",
      "Downloading file:  stories\\026.txt\n",
      "File  026.txt  already exists.\n",
      "Downloading file:  stories\\027.txt\n",
      "File  027.txt  already exists.\n",
      "Downloading file:  stories\\028.txt\n",
      "File  028.txt  already exists.\n",
      "Downloading file:  stories\\029.txt\n",
      "File  029.txt  already exists.\n",
      "Downloading file:  stories\\030.txt\n",
      "File  030.txt  already exists.\n",
      "Downloading file:  stories\\031.txt\n",
      "File  031.txt  already exists.\n",
      "Downloading file:  stories\\032.txt\n",
      "File  032.txt  already exists.\n",
      "Downloading file:  stories\\033.txt\n",
      "File  033.txt  already exists.\n",
      "Downloading file:  stories\\034.txt\n",
      "File  034.txt  already exists.\n",
      "Downloading file:  stories\\035.txt\n",
      "File  035.txt  already exists.\n",
      "Downloading file:  stories\\036.txt\n",
      "File  036.txt  already exists.\n",
      "Downloading file:  stories\\037.txt\n",
      "File  037.txt  already exists.\n",
      "Downloading file:  stories\\038.txt\n",
      "File  038.txt  already exists.\n",
      "Downloading file:  stories\\039.txt\n",
      "File  039.txt  already exists.\n",
      "Downloading file:  stories\\040.txt\n",
      "File  040.txt  already exists.\n",
      "Downloading file:  stories\\041.txt\n",
      "File  041.txt  already exists.\n",
      "Downloading file:  stories\\042.txt\n",
      "File  042.txt  already exists.\n",
      "Downloading file:  stories\\043.txt\n",
      "File  043.txt  already exists.\n",
      "Downloading file:  stories\\044.txt\n",
      "File  044.txt  already exists.\n",
      "Downloading file:  stories\\045.txt\n",
      "File  045.txt  already exists.\n",
      "Downloading file:  stories\\046.txt\n",
      "File  046.txt  already exists.\n",
      "Downloading file:  stories\\047.txt\n",
      "File  047.txt  already exists.\n",
      "Downloading file:  stories\\048.txt\n",
      "File  048.txt  already exists.\n",
      "Downloading file:  stories\\049.txt\n",
      "File  049.txt  already exists.\n",
      "Downloading file:  stories\\050.txt\n",
      "File  050.txt  already exists.\n",
      "Downloading file:  stories\\051.txt\n",
      "File  051.txt  already exists.\n",
      "Downloading file:  stories\\052.txt\n",
      "File  052.txt  already exists.\n",
      "Downloading file:  stories\\053.txt\n",
      "File  053.txt  already exists.\n",
      "Downloading file:  stories\\054.txt\n",
      "File  054.txt  already exists.\n",
      "Downloading file:  stories\\055.txt\n",
      "File  055.txt  already exists.\n",
      "Downloading file:  stories\\056.txt\n",
      "File  056.txt  already exists.\n",
      "Downloading file:  stories\\057.txt\n",
      "File  057.txt  already exists.\n",
      "Downloading file:  stories\\058.txt\n",
      "File  058.txt  already exists.\n",
      "Downloading file:  stories\\059.txt\n",
      "File  059.txt  already exists.\n",
      "Downloading file:  stories\\060.txt\n",
      "File  060.txt  already exists.\n",
      "Downloading file:  stories\\061.txt\n",
      "File  061.txt  already exists.\n",
      "Downloading file:  stories\\062.txt\n",
      "File  062.txt  already exists.\n",
      "Downloading file:  stories\\063.txt\n",
      "File  063.txt  already exists.\n",
      "Downloading file:  stories\\064.txt\n",
      "File  064.txt  already exists.\n",
      "Downloading file:  stories\\065.txt\n",
      "File  065.txt  already exists.\n",
      "Downloading file:  stories\\066.txt\n",
      "File  066.txt  already exists.\n",
      "Downloading file:  stories\\067.txt\n",
      "File  067.txt  already exists.\n",
      "Downloading file:  stories\\068.txt\n",
      "File  068.txt  already exists.\n",
      "Downloading file:  stories\\069.txt\n",
      "File  069.txt  already exists.\n",
      "Downloading file:  stories\\070.txt\n",
      "File  070.txt  already exists.\n",
      "Downloading file:  stories\\071.txt\n",
      "File  071.txt  already exists.\n",
      "Downloading file:  stories\\072.txt\n",
      "File  072.txt  already exists.\n",
      "Downloading file:  stories\\073.txt\n",
      "File  073.txt  already exists.\n",
      "Downloading file:  stories\\074.txt\n",
      "File  074.txt  already exists.\n",
      "Downloading file:  stories\\075.txt\n",
      "File  075.txt  already exists.\n",
      "Downloading file:  stories\\076.txt\n",
      "File  076.txt  already exists.\n",
      "Downloading file:  stories\\077.txt\n",
      "File  077.txt  already exists.\n",
      "Downloading file:  stories\\078.txt\n",
      "File  078.txt  already exists.\n",
      "Downloading file:  stories\\079.txt\n",
      "File  079.txt  already exists.\n",
      "Downloading file:  stories\\080.txt\n",
      "File  080.txt  already exists.\n",
      "Downloading file:  stories\\081.txt\n",
      "File  081.txt  already exists.\n",
      "Downloading file:  stories\\082.txt\n",
      "File  082.txt  already exists.\n",
      "Downloading file:  stories\\083.txt\n",
      "File  083.txt  already exists.\n",
      "Downloading file:  stories\\084.txt\n",
      "File  084.txt  already exists.\n",
      "Downloading file:  stories\\085.txt\n",
      "File  085.txt  already exists.\n",
      "Downloading file:  stories\\086.txt\n",
      "File  086.txt  already exists.\n",
      "Downloading file:  stories\\087.txt\n",
      "File  087.txt  already exists.\n",
      "Downloading file:  stories\\088.txt\n",
      "File  088.txt  already exists.\n",
      "Downloading file:  stories\\089.txt\n",
      "File  089.txt  already exists.\n",
      "Downloading file:  stories\\090.txt\n",
      "File  090.txt  already exists.\n",
      "Downloading file:  stories\\091.txt\n",
      "File  091.txt  already exists.\n",
      "Downloading file:  stories\\092.txt\n",
      "File  092.txt  already exists.\n",
      "Downloading file:  stories\\093.txt\n",
      "File  093.txt  already exists.\n",
      "Downloading file:  stories\\094.txt\n",
      "File  094.txt  already exists.\n",
      "Downloading file:  stories\\095.txt\n",
      "File  095.txt  already exists.\n",
      "Downloading file:  stories\\096.txt\n",
      "File  096.txt  already exists.\n",
      "Downloading file:  stories\\097.txt\n",
      "File  097.txt  already exists.\n",
      "Downloading file:  stories\\098.txt\n",
      "File  098.txt  already exists.\n",
      "Downloading file:  stories\\099.txt\n",
      "File  099.txt  already exists.\n",
      "Downloading file:  stories\\100.txt\n",
      "File  100.txt  already exists.\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.cs.cmu.edu/~spok/grimmtmp/'\n",
    "\n",
    "# Create a directory if needed\n",
    "dir_name = 'stories'\n",
    "if not os.path.exists(dir_name):\n",
    "    os.mkdir(dir_name)\n",
    "    \n",
    "def maybe_download(filename):\n",
    "  \"\"\"Download a file if not present\"\"\"\n",
    "  print('Downloading file: ', dir_name+ os.sep+filename)\n",
    "    \n",
    "  if not os.path.exists(dir_name+os.sep+filename):\n",
    "    filename, _ = urlretrieve(url + filename, dir_name+os.sep+filename)\n",
    "  else:\n",
    "    print('File ',filename, ' already exists.')\n",
    "  \n",
    "  return filename\n",
    "\n",
    "num_files = 100\n",
    "filenames = [format(i, '03d')+'.txt' for i in range(1,101)]\n",
    "\n",
    "for fn in filenames:\n",
    "    maybe_download(fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check if the files are downloaded. \n",
    "There should be 100 files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 files found.\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(filenames)):\n",
    "    file_exists = os.path.isfile(os.path.join(dir_name,filenames[i]))\n",
    "    assert file_exists\n",
    "print('%d files found.'%len(filenames))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading data\n",
    "Data will be stored in a list of lists where the each list represents a document and document is a list of words. We will then break the text into bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file stories\\001.txt\n",
      "Data size (Characters) (Document 0) 3667\n",
      "Sample string (Document 0) ['in', ' o', 'ld', 'en', ' t', 'im', 'es', ' w', 'he', 'n ', 'wi', 'sh', 'in', 'g ', 'st', 'il', 'l ', 'he', 'lp', 'ed', ' o', 'ne', ', ', 'th', 'er', 'e ', 'li', 've', 'd ', 'a ', 'ki', 'ng', '\\nw', 'ho', 'se', ' d', 'au', 'gh', 'te', 'rs', ' w', 'er', 'e ', 'al', 'l ', 'be', 'au', 'ti', 'fu', 'l,']\n",
      "\n",
      "Processing file stories\\002.txt\n",
      "Data size (Characters) (Document 1) 4928\n",
      "Sample string (Document 1) ['ha', 'rd', ' b', 'y ', 'a ', 'gr', 'ea', 't ', 'fo', 're', 'st', ' d', 'we', 'lt', ' a', ' w', 'oo', 'd-', 'cu', 'tt', 'er', ' w', 'it', 'h ', 'hi', 's ', 'wi', 'fe', ', ', 'wh', 'o ', 'ha', 'd ', 'an', '\\no', 'nl', 'y ', 'ch', 'il', 'd,', ' a', ' l', 'it', 'tl', 'e ', 'gi', 'rl', ' t', 'hr', 'ee']\n",
      "\n",
      "Processing file stories\\003.txt\n",
      "Data size (Characters) (Document 2) 9745\n",
      "Sample string (Document 2) ['a ', 'ce', 'rt', 'ai', 'n ', 'fa', 'th', 'er', ' h', 'ad', ' t', 'wo', ' s', 'on', 's,', ' t', 'he', ' e', 'ld', 'er', ' o', 'f ', 'wh', 'om', ' w', 'as', ' s', 'ma', 'rt', ' a', 'nd', '\\ns', 'en', 'si', 'bl', 'e,', ' a', 'nd', ' c', 'ou', 'ld', ' d', 'o ', 'ev', 'er', 'yt', 'hi', 'ng', ', ', 'bu']\n",
      "\n",
      "Processing file stories\\004.txt\n",
      "Data size (Characters) (Document 3) 2852\n",
      "Sample string (Document 3) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', 'n ', 'ol', 'd ', 'go', 'at', ' w', 'ho', ' h', 'ad', ' s', 'ev', 'en', ' l', 'it', 'tl', 'e ', 'ki', 'ds', ', ', 'an', 'd\\n', 'lo', 've', 'd ', 'th', 'em', ' w', 'it', 'h ', 'al', 'l ', 'th', 'e ', 'lo', 've', ' o']\n",
      "\n",
      "Processing file stories\\005.txt\n",
      "Data size (Characters) (Document 4) 8189\n",
      "Sample string (Document 4) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', 'n ', 'ol', 'd ', 'ki', 'ng', ' w', 'ho', ' w', 'as', ' i', 'll', ' a', 'nd', ' t', 'ho', 'ug', 'ht', ' t', 'o\\n', 'hi', 'ms', 'el', 'f ', \"'i\", ' a', 'm ', 'ly', 'in', 'g ', 'on', ' w', 'ha', 't ', 'mu', 'st', ' b']\n",
      "\n",
      "Processing file stories\\006.txt\n",
      "Data size (Characters) (Document 5) 4369\n",
      "Sample string (Document 5) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'ea', 'sa', 'nt', ' w', 'ho', ' h', 'ad', ' d', 'ri', 've', 'n ', 'hi', 's ', 'co', 'w ', 'to', ' t', 'he', ' f', 'ai', 'r,', ' a', 'nd', ' s', 'ol', 'd\\n', 'he', 'r ', 'fo', 'r ', 'se', 've', 'n ', 'ta', 'le', 'rs', '. ', ' o', 'n ', 'th', 'e ']\n",
      "\n",
      "Processing file stories\\007.txt\n",
      "Data size (Characters) (Document 6) 5216\n",
      "Sample string (Document 6) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'a ', 'ki', 'ng', ' a', 'nd', ' a', ' q', 'ue', 'en', ' w', 'ho', ' l', 'iv', 'ed', '\\nh', 'ap', 'pi', 'ly', ' t', 'og', 'et', 'he', 'r ', 'an', 'd ', 'ha', 'd ', 'tw', 'el', 've', ' c', 'hi', 'ld', 're', 'n,', ' b']\n",
      "\n",
      "Processing file stories\\008.txt\n",
      "Data size (Characters) (Document 7) 6097\n",
      "Sample string (Document 7) ['li', 'tt', 'le', ' b', 'ro', 'th', 'er', ' t', 'oo', 'k ', 'hi', 's ', 'li', 'tt', 'le', ' s', 'is', 'te', 'r ', 'by', ' t', 'he', ' h', 'an', 'd ', 'an', 'd ', 'sa', 'id', ', ', 'si', 'nc', 'e\\n', 'ou', 'r ', 'mo', 'th', 'er', ' d', 'ie', 'd ', 'we', ' h', 'av', 'e ', 'ha', 'd ', 'no', ' h', 'ap']\n",
      "\n",
      "Processing file stories\\009.txt\n",
      "Data size (Characters) (Document 8) 3699\n",
      "Sample string (Document 8) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'a ', 'ma', 'n ', 'an', 'd ', 'a ', 'wo', 'ma', 'n ', 'wh', 'o ', 'ha', 'd ', 'lo', 'ng', ' i', 'n ', 'va', 'in', '\\nw', 'is', 'he', 'd ', 'fo', 'r ', 'a ', 'ch', 'il', 'd.', '  ', 'at', ' l', 'en', 'gt', 'h ', 'th', 'e ', 'wo', 'ma', 'n ', 'ho', 'pe']\n",
      "\n",
      "Processing file stories\\010.txt\n",
      "Data size (Characters) (Document 9) 5268\n",
      "Sample string (Document 9) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', 'se', ' w', 'if', 'e ', 'di', 'ed', ', ', 'an', 'd ', 'a ', 'wo', 'ma', 'n ', 'wh', 'os', 'e ', 'hu', 'sb', 'an', 'd\\n', 'di', 'ed', ', ', 'an', 'd ', 'th', 'e ', 'ma', 'n ', 'ha', 'd ', 'a ', 'da', 'ug', 'ht', 'er', ', ', 'an']\n",
      "\n",
      "Processing file stories\\011.txt\n",
      "Data size (Characters) (Document 10) 2377\n",
      "Sample string (Document 10) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' g', 'ir', 'l ', 'wh', 'o ', 'wa', 's ', 'id', 'le', ' a', 'nd', ' w', 'ou', 'ld', ' n', 'ot', ' s', 'pi', 'n,', ' a', 'nd', '\\nl', 'et', ' h', 'er', ' m', 'ot', 'he', 'r ', 'sa', 'y ', 'wh', 'at', ' s', 'he', ' w', 'ou', 'ld', ', ', 'sh', 'e ', 'co']\n",
      "\n",
      "Processing file stories\\012.txt\n",
      "Data size (Characters) (Document 11) 7695\n",
      "Sample string (Document 11) ['ha', 'rd', ' b', 'y ', 'a ', 'gr', 'ea', 't ', 'fo', 're', 'st', ' d', 'we', 'lt', ' a', ' p', 'oo', 'r ', 'wo', 'od', '-c', 'ut', 'te', 'r ', 'wi', 'th', ' h', 'is', ' w', 'if', 'e\\n', 'an', 'd ', 'hi', 's ', 'tw', 'o ', 'ch', 'il', 'dr', 'en', '. ', ' t', 'he', ' b', 'oy', ' w', 'as', ' c', 'al']\n",
      "\n",
      "Processing file stories\\013.txt\n",
      "Data size (Characters) (Document 12) 3665\n",
      "Sample string (Document 12) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' o', 'n ', 'a ', 'ti', 'me', ' a', ' p', 'oo', 'r ', 'ma', 'n,', ' w', 'ho', ' c', 'ou', 'ld', ' n', 'o ', 'lo', 'ng', 'er', '\\ns', 'up', 'po', 'rt', ' h', 'is', ' o', 'nl', 'y ', 'so', 'n.', '  ', 'th', 'en', ' s', 'ai', 'd ', 'th', 'e ', 'so', 'n,', ' d']\n",
      "\n",
      "Processing file stories\\014.txt\n",
      "Data size (Characters) (Document 13) 4178\n",
      "Sample string (Document 13) ['a ', 'lo', 'ng', ' t', 'im', 'e ', 'ag', 'o ', 'th', 'er', 'e ', 'li', 've', 'd ', 'a ', 'ki', 'ng', ' w', 'ho', ' w', 'as', ' f', 'am', 'ed', ' f', 'or', ' h', 'is', ' w', 'is', 'do', 'm\\n', 'th', 'ro', 'ug', 'h ', 'al', 'l ', 'th', 'e ', 'la', 'nd', '. ', ' n', 'ot', 'hi', 'ng', ' w', 'as', ' h']\n",
      "\n",
      "Processing file stories\\015.txt\n",
      "Data size (Characters) (Document 14) 8674\n",
      "Sample string (Document 14) ['on', 'e ', 'su', 'mm', 'er', \"'s\", ' m', 'or', 'ni', 'ng', ' a', ' l', 'it', 'tl', 'e ', 'ta', 'il', 'or', ' w', 'as', ' s', 'it', 'ti', 'ng', ' o', 'n ', 'hi', 's ', 'ta', 'bl', 'e\\n', 'by', ' t', 'he', ' w', 'in', 'do', 'w,', ' h', 'e ', 'wa', 's ', 'in', ' g', 'oo', 'd ', 'sp', 'ir', 'it', 's,']\n",
      "\n",
      "Processing file stories\\016.txt\n",
      "Data size (Characters) (Document 15) 7018\n",
      "Sample string (Document 15) ['\\tc', 'in', 'de', 're', 'll', 'a\\n', 'th', 'e ', 'wi', 'fe', ' o', 'f ', 'a ', 'ri', 'ch', ' m', 'an', ' f', 'el', 'l ', 'si', 'ck', ', ', 'an', 'd ', 'as', ' s', 'he', ' f', 'el', 't ', 'th', 'at', ' h', 'er', ' e', 'nd', '\\nw', 'as', ' d', 'ra', 'wi', 'ng', ' n', 'ea', 'r,', ' s', 'he', ' c', 'al']\n",
      "\n",
      "Processing file stories\\017.txt\n",
      "Data size (Characters) (Document 16) 3039\n",
      "Sample string (Document 16) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', \"g'\", 's ', 'so', 'n ', 'wh', 'o ', 'wa', 's ', 'se', 'iz', 'ed', ' w', 'it', 'h ', 'a ', 'de', 'si', 're', ' t', 'o ', 'tr', 'av', 'el', '\\na', 'bo', 'ut', ' t', 'he', ' w', 'or', 'ld', ', ', 'an', 'd ', 'to', 'ok', ' n', 'o ', 'on', 'e ']\n",
      "\n",
      "Processing file stories\\018.txt\n",
      "Data size (Characters) (Document 17) 3020\n",
      "Sample string (Document 17) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'id', 'ow', ' w', 'ho', ' h', 'ad', ' t', 'wo', ' d', 'au', 'gh', 'te', 'rs', ' -', ' o', 'ne', ' o', 'f\\n', 'wh', 'om', ' w', 'as', ' p', 're', 'tt', 'y ', 'an', 'd ', 'in', 'du', 'st', 'ri', 'ou', 's,', ' w', 'hi', 'ls', 't ', 'th', 'e ', 'ot']\n",
      "\n",
      "Processing file stories\\019.txt\n",
      "Data size (Characters) (Document 18) 2465\n",
      "Sample string (Document 18) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' h', 'ad', ' s', 'ev', 'en', ' s', 'on', 's,', ' a', 'nd', ' s', 'ti', 'll', ' h', 'e ', 'ha', 'd\\n', 'no', ' d', 'au', 'gh', 'te', 'r,', ' h', 'ow', 'ev', 'er', ' m', 'uc', 'h ', 'he', ' w', 'is', 'he', 'd ', 'fo', 'r ', 'on']\n",
      "\n",
      "Processing file stories\\020.txt\n",
      "Data size (Characters) (Document 19) 3703\n",
      "Sample string (Document 19) ['\\tl', 'it', 'tl', 'e ', 're', 'd-', 'ca', 'p\\n', '\\no', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'th', 'er', 'e ', 'wa', 's ', 'a ', 'de', 'ar', ' l', 'it', 'tl', 'e ', 'gi', 'rl', ' w', 'ho', ' w', 'as', ' l', 'ov', 'ed', '\\nb', 'y ', 'ev', 'er', 'y ', 'on', 'e ', 'wh', 'o ', 'lo', 'ok', 'ed']\n",
      "\n",
      "Processing file stories\\021.txt\n",
      "Data size (Characters) (Document 20) 1924\n",
      "Sample string (Document 20) ['in', ' a', ' c', 'er', 'ta', 'in', ' c', 'ou', 'nt', 'ry', ' t', 'he', 're', ' w', 'as', ' o', 'nc', 'e ', 'gr', 'ea', 't ', 'la', 'me', 'nt', 'at', 'io', 'n ', 'ov', 'er', ' a', '\\nw', 'il', 'd ', 'bo', 'ar', ' t', 'ha', 't ', 'la', 'id', ' w', 'as', 'te', ' t', 'he', ' f', 'ar', 'me', \"r'\", 's ']\n",
      "\n",
      "Processing file stories\\022.txt\n",
      "Data size (Characters) (Document 21) 6561\n",
      "Sample string (Document 21) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'wo', 'ma', 'n ', 'wh', 'o ', 'ga', 've', ' b', 'ir', 'th', ' t', 'o ', 'a ', 'li', 'tt', 'le', ' s', 'on', ',\\n', 'an', 'd ', 'as', ' h', 'e ', 'ca', 'me', ' i', 'nt', 'o ', 'th', 'e ', 'wo', 'rl', 'd ', 'wi', 'th', ' a', ' c', 'au']\n",
      "\n",
      "Processing file stories\\023.txt\n",
      "Data size (Characters) (Document 22) 5956\n",
      "Sample string (Document 22) ['a ', 'ce', 'rt', 'ai', 'n ', 'mi', 'll', 'er', ' h', 'ad', ' l', 'it', 'tl', 'e ', 'by', ' l', 'it', 'tl', 'e ', 'fa', 'll', 'en', ' i', 'nt', 'o ', 'po', 've', 'rt', 'y,', ' a', 'nd', '\\nh', 'ad', ' n', 'ot', 'hi', 'ng', ' l', 'ef', 't ', 'bu', 't ', 'hi', 's ', 'mi', 'll', ' a', 'nd', ' a', ' l']\n",
      "\n",
      "Processing file stories\\024.txt\n",
      "Data size (Characters) (Document 23) 2529\n",
      "Sample string (Document 23) ['th', 'e ', 'mo', 'th', 'er', ' o', 'f ', 'ha', 'ns', ' s', 'ai', 'd,', ' w', 'hi', 'th', 'er', ' a', 'wa', 'y,', ' h', 'an', 's.', '  ', 'ha', 'ns', ' a', 'ns', 'we', 're', 'd,', ' t', 'o\\n', 'gr', 'et', 'el', '. ', ' b', 'eh', 'av', 'e ', 'we', 'll', ', ', 'ha', 'ns', '. ', ' o', 'h,', ' i', \"'l\"]\n",
      "\n",
      "Processing file stories\\025.txt\n",
      "Data size (Characters) (Document 24) 2416\n",
      "Sample string (Document 24) ['an', ' a', 'ge', 'd ', 'co', 'un', 't ', 'on', 'ce', ' l', 'iv', 'ed', ' i', 'n ', 'sw', 'it', 'ze', 'rl', 'an', 'd,', ' w', 'ho', ' h', 'ad', ' a', 'n ', 'on', 'ly', ' s', 'on', ',\\n', 'bu', 't ', 'he', ' w', 'as', ' s', 'tu', 'pi', 'd,', ' a', 'nd', ' c', 'ou', 'ld', ' l', 'ea', 'rn', ' n', 'ot']\n",
      "\n",
      "Processing file stories\\026.txt\n",
      "Data size (Characters) (Document 25) 3369\n",
      "Sample string (Document 25) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' h', 'ad', ' a', ' d', 'au', 'gh', 'te', 'r ', 'wh', 'o ', 'wa', 's ', 'ca', 'll', 'ed', ' c', 'le', 've', 'r\\n', 'el', 'si', 'e.', '  ', 'an', 'd ', 'wh', 'en', ' s', 'he', ' h', 'ad', ' g', 'ro', 'wn', ' u', 'p ', 'he', 'r ']\n",
      "\n",
      "Processing file stories\\027.txt\n",
      "Data size (Characters) (Document 26) 10013\n",
      "Sample string (Document 26) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' t', 'ai', 'lo', 'r ', 'wh', 'o ', 'ha', 'd ', 'th', 're', 'e ', 'so', 'ns', ', ', 'an', 'd\\n', 'on', 'ly', ' o', 'ne', ' g', 'oa', 't.', '  ', 'bu', 't ', 'as', ' t', 'he', ' g', 'oa', 't ', 'su', 'pp', 'or', 'te']\n",
      "\n",
      "Processing file stories\\028.txt\n",
      "Data size (Characters) (Document 27) 5788\n",
      "Sample string (Document 27) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'pe', 'as', 'an', 't ', 'wh', 'o ', 'sa', 't ', 'in', ' t', 'he', ' e', 've', 'ni', 'ng', ' b', 'y ', 'th', 'e\\n', 'he', 'ar', 'th', ' a', 'nd', ' p', 'ok', 'ed', ' t', 'he', ' f', 'ir', 'e,', ' a', 'nd', ' h', 'is', ' w', 'if', 'e ']\n",
      "\n",
      "Processing file stories\\029.txt\n",
      "Data size (Characters) (Document 28) 1335\n",
      "Sample string (Document 28) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'se', 'rv', 'an', 't-', 'gi', 'rl', ' w', 'ho', ' w', 'as', ' i', 'nd', 'us', 'tr', 'io', 'us', ' a', 'nd', ' c', 'le', 'an', 'ly', '\\na', 'nd', ' s', 'we', 'pt', ' t', 'he', ' h', 'ou', 'se', ' e', 've', 'ry', ' d', 'ay', ', ', 'an']\n",
      "\n",
      "Processing file stories\\030.txt\n",
      "Data size (Characters) (Document 29) 3591\n",
      "Sample string (Document 29) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' m', 'il', 'le', 'r,', ' w', 'ho', ' h', 'ad', ' a', ' b', 'ea', 'ut', 'if', 'ul', '\\nd', 'au', 'gh', 'te', 'r,', ' a', 'nd', ' a', 's ', 'sh', 'e ', 'wa', 's ', 'gr', 'ow', 'n ', 'up', ', ', 'he', ' w', 'is', 'he']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file stories\\031.txt\n",
      "Data size (Characters) (Document 30) 1624\n",
      "Sample string (Document 30) ['a ', 'po', 'or', ' m', 'an', ' h', 'ad', ' s', 'o ', 'ma', 'ny', ' c', 'hi', 'ld', 're', 'n ', 'th', 'at', ' h', 'e ', 'ha', 'd ', 'al', 're', 'ad', 'y ', 'as', 'ke', 'd\\n', 'ev', 'er', 'yo', 'ne', ' i', 'n ', 'th', 'e ', 'wo', 'rl', 'd ', 'to', ' b', 'e ', 'go', 'df', 'at', 'he', 'r,', ' a', 'nd']\n",
      "\n",
      "Processing file stories\\032.txt\n",
      "Data size (Characters) (Document 31) 758\n",
      "Sample string (Document 31) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' l', 'it', 'tl', 'e ', 'gi', 'rl', ' w', 'ho', ' w', 'as', ' o', 'bs', 'ti', 'na', 'te', ' a', 'nd', ' i', 'nq', 'ui', 'si', 'ti', 've', ',\\n', 'an', 'd ', 'wh', 'en', ' h', 'er', ' p', 'ar', 'en', 'ts', ' t', 'ol', 'd ', 'he', 'r ', 'to', ' d', 'o ']\n",
      "\n",
      "Processing file stories\\033.txt\n",
      "Data size (Characters) (Document 32) 3121\n",
      "Sample string (Document 32) ['a ', 'po', 'or', ' m', 'an', ' h', 'ad', ' t', 'we', 'lv', 'e ', 'ch', 'il', 'dr', 'en', ' a', 'nd', ' w', 'as', ' f', 'or', 'ce', 'd ', 'to', ' w', 'or', 'k ', 'ni', 'gh', 't ', 'an', 'd\\n', 'da', 'y ', 'to', ' g', 'iv', 'e ', 'th', 'em', ' e', 've', 'n ', 'br', 'ea', 'd.', '  ', 'wh', 'en', ' t']\n",
      "\n",
      "Processing file stories\\034.txt\n",
      "Data size (Characters) (Document 33) 4192\n",
      "Sample string (Document 33) ['a ', 'ce', 'rt', 'ai', 'n ', 'ta', 'il', 'or', ' h', 'ad', ' a', ' s', 'on', ', ', 'wh', 'o ', 'ha', 'pp', 'en', 'ed', ' t', 'o ', 'be', ' s', 'ma', 'll', ', ', 'an', 'd\\n', 'no', ' b', 'ig', 'ge', 'r ', 'th', 'an', ' a', ' t', 'hu', 'mb', ', ', 'an', 'd ', 'on', ' t', 'hi', 's ', 'ac', 'co', 'un']\n",
      "\n",
      "Processing file stories\\035.txt\n",
      "Data size (Characters) (Document 34) 3650\n",
      "Sample string (Document 34) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'iz', 'ar', 'd ', 'wh', 'o ', 'us', 'ed', ' t', 'o ', 'ta', 'ke', ' t', 'he', ' f', 'or', 'm ', 'of', ' a', ' p', 'oo', 'r\\n', 'ma', 'n,', ' a', 'nd', ' w', 'en', 't ', 'to', ' h', 'ou', 'se', 's ', 'an', 'd ', 'be', 'gg', 'ed', ', ', 'an', 'd ']\n",
      "\n",
      "Processing file stories\\036.txt\n",
      "Data size (Characters) (Document 35) 8219\n",
      "Sample string (Document 35) ['it', ' i', 's ', 'no', 'w ', 'lo', 'ng', ' a', 'go', ', ', 'qu', 'it', 'e ', 'tw', 'o ', 'th', 'ou', 'sa', 'nd', ' y', 'ea', 'rs', ', ', 'si', 'nc', 'e ', 'th', 'er', 'e ', 'wa', 's\\n', 'a ', 'ri', 'ch', ' m', 'an', ' w', 'ho', ' h', 'ad', ' a', ' b', 'ea', 'ut', 'if', 'ul', ' a', 'nd', ' p', 'io']\n",
      "\n",
      "Processing file stories\\037.txt\n",
      "Data size (Characters) (Document 36) 2151\n",
      "Sample string (Document 36) ['a ', 'fa', 'rm', 'er', ' o', 'nc', 'e ', 'ha', 'd ', 'a ', 'fa', 'it', 'hf', 'ul', ' d', 'og', ' c', 'al', 'le', 'd ', 'su', 'lt', 'an', ', ', 'wh', 'o ', 'ha', 'd ', 'gr', 'ow', 'n\\n', 'ol', 'd,', ' a', 'nd', ' l', 'os', 't ', 'al', 'l ', 'hi', 's ', 'te', 'et', 'h,', ' s', 'o ', 'th', 'at', ' h']\n",
      "\n",
      "Processing file stories\\038.txt\n",
      "Data size (Characters) (Document 37) 5129\n",
      "Sample string (Document 37) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ', ', 'a ', 'ce', 'rt', 'ai', 'n ', 'ki', 'ng', ' w', 'as', ' h', 'un', 'ti', 'ng', ' i', 'n ', 'a ', 'gr', 'ea', 't ', 'fo', 're', 'st', ',\\n', 'an', 'd ', 'he', ' c', 'ha', 'se', 'd ', 'a ', 'wi', 'ld', ' b', 'ea', 'st', ' s', 'o ', 'ea', 'ge', 'rl']\n",
      "\n",
      "Processing file stories\\039.txt\n",
      "Data size (Characters) (Document 38) 3472\n",
      "Sample string (Document 38) ['\\tb', 'ri', 'ar', '-r', 'os', 'e\\n', '\\na', ' l', 'on', 'g ', 'ti', 'me', ' a', 'go', ' t', 'he', 're', ' w', 'er', 'e ', 'a ', 'ki', 'ng', ' a', 'nd', ' q', 'ue', 'en', ' w', 'ho', ' s', 'ai', 'd ', 'ev', 'er', 'y\\n', 'da', 'y,', ' a', 'h,', ' i', 'f ', 'on', 'ly', ' w', 'e ', 'ha', 'd ', 'a ', 'ch']\n",
      "\n",
      "Processing file stories\\040.txt\n",
      "Data size (Characters) (Document 39) 2490\n",
      "Sample string (Document 39) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' f', 'or', 'es', 'te', 'r ', 'wh', 'o ', 'we', 'nt', ' i', 'nt', 'o ', 'th', 'e ', 'fo', 're', 'st', ' t', 'o ', 'hu', 'nt', ',\\n', 'an', 'd ', 'as', ' h', 'e ', 'en', 'te', 're', 'd ', 'it', ' h', 'e ', 'he', 'ar', 'd ', 'a ', 'so', 'un', 'd ', 'of']\n",
      "\n",
      "Processing file stories\\041.txt\n",
      "Data size (Characters) (Document 40) 4273\n",
      "Sample string (Document 40) ['a ', 'ki', 'ng', ' h', 'ad', ' a', ' d', 'au', 'gh', 'te', 'r ', 'wh', 'o ', 'wa', 's ', 'be', 'au', 'ti', 'fu', 'l ', 'be', 'yo', 'nd', ' a', 'll', ' m', 'ea', 'su', 're', ',\\n', 'bu', 't ', 'so', ' p', 'ro', 'ud', ' a', 'nd', ' h', 'au', 'gh', 'ty', ' w', 'it', 'ha', 'l ', 'th', 'at', ' n', 'o ']\n",
      "\n",
      "Processing file stories\\042.txt\n",
      "Data size (Characters) (Document 41) 8327\n",
      "Sample string (Document 41) ['\\ts', 'no', 'w ', 'wh', 'it', 'e ', 'an', 'd ', 'th', 'e ', 'se', 've', 'n ', 'dw', 'ar', 'fs', '\\n\\n', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' i', 'n ', 'th', 'e ', 'mi', 'dd', 'le', ' o', 'f ', 'wi', 'nt', 'er', ', ', 'wh', 'en', ' t', 'he', ' f', 'la', 'ke', 's ', 'of', '\\ns', 'no', 'w ']\n",
      "\n",
      "Processing file stories\\043.txt\n",
      "Data size (Characters) (Document 42) 6128\n",
      "Sample string (Document 42) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'th', 're', 'e ', 'br', 'ot', 'he', 'rs', ' w', 'ho', ' h', 'ad', ' f', 'al', 'le', 'n ', 'de', 'ep', 'er', ' a', 'nd', ' d', 'ee', 'pe', 'r ', 'in', 'to', '\\np', 'ov', 'er', 'ty', ', ', 'an', 'd ', 'at', ' l', 'as', 't ', 'th', 'ei', 'r ', 'ne', 'ed']\n",
      "\n",
      "Processing file stories\\044.txt\n",
      "Data size (Characters) (Document 43) 2819\n",
      "Sample string (Document 43) ['\\tr', 'um', 'pe', 'ls', 'ti', 'lt', 'sk', 'in', '\\n\\n', 'on', 'ce', ' t', 'he', 're', ' w', 'as', ' a', ' m', 'il', 'le', 'r ', 'wh', 'o ', 'wa', 's ', 'po', 'or', ', ', 'bu', 't ', 'wh', 'o ', 'ha', 'd ', 'a ', 'be', 'au', 'ti', 'fu', 'l\\n', 'da', 'ug', 'ht', 'er', '. ', ' n', 'ow', ' i', 't ', 'ha']\n",
      "\n",
      "Processing file stories\\045.txt\n",
      "Data size (Characters) (Document 44) 3822\n",
      "Sample string (Document 44) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' w', 'om', 'an', ' w', 'ho', ' w', 'as', ' a', ' r', 'ea', 'l ', 'wi', 'tc', 'h ', 'an', 'd ', 'ha', 'd ', 'tw', 'o\\n', 'da', 'ug', 'ht', 'er', 's,', ' o', 'ne', ' u', 'gl', 'y ', 'an', 'd ', 'wi', 'ck', 'ed', ', ']\n",
      "\n",
      "Processing file stories\\046.txt\n",
      "Data size (Characters) (Document 45) 7772\n",
      "Sample string (Document 45) ['in', ' o', 'ld', 'en', ' t', 'im', 'es', ' t', 'he', 're', ' w', 'as', ' a', ' k', 'in', 'g,', ' w', 'ho', ' h', 'ad', ' b', 'eh', 'in', 'd ', 'hi', 's ', 'pa', 'la', 'ce', ' a', '\\nb', 'ea', 'ut', 'if', 'ul', ' p', 'le', 'as', 'ur', 'e-', 'ga', 'rd', 'en', ' i', 'n ', 'wh', 'ic', 'h ', 'th', 'er']\n",
      "\n",
      "Processing file stories\\047.txt\n",
      "Data size (Characters) (Document 46) 22158\n",
      "Sample string (Document 46) ['th', 'er', 'e ', 'we', 're', ' o', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'tw', 'o ', 'br', 'ot', 'he', 'rs', ', ', 'on', 'e ', 'ri', 'ch', ' a', 'nd', ' t', 'he', ' o', 'th', 'er', '\\np', 'oo', 'r.', '  ', 'th', 'e ', 'ri', 'ch', ' o', 'ne', ' w', 'as', ' a', ' g', 'ol', 'ds', 'mi', 'th']\n",
      "\n",
      "Processing file stories\\048.txt\n",
      "Data size (Characters) (Document 47) 2169\n",
      "Sample string (Document 47) ['tw', 'o ', 'ki', 'ng', \"s'\", ' s', 'on', 's ', 'on', 'ce', ' w', 'en', 't ', 'ou', 't ', 'in', ' s', 'ea', 'rc', 'h ', 'of', ' a', 'dv', 'en', 'tu', 're', 's,', ' a', 'nd', ' f', 'el', 'l ', 'in', 'to', '\\na', ' w', 'il', 'd,', ' d', 'is', 'or', 'de', 'rl', 'y ', 'wa', 'y ', 'of', ' l', 'iv', 'in']\n",
      "\n",
      "Processing file stories\\049.txt\n",
      "Data size (Characters) (Document 48) 2822\n",
      "Sample string (Document 48) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' k', 'in', 'g ', 'wh', 'o ', 'ha', 'd ', 'th', 're', 'e ', 'so', 'ns', ', ', 'of', ' w', 'ho', 'm ', 'tw', 'o\\n', 'we', 're', ' c', 'le', 've', 'r ', 'an', 'd ', 'wi', 'se', ', ', 'bu', 't ', 'th', 'e ', 'th', 'ir']\n",
      "\n",
      "Processing file stories\\050.txt\n",
      "Data size (Characters) (Document 49) 4034\n",
      "Sample string (Document 49) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'ma', 'n ', 'wh', 'o ', 'ha', 'd ', 'th', 're', 'e ', 'so', 'ns', ', ', 'th', 'e ', 'yo', 'un', 'ge', 'st', ' o', 'f ', 'wh', 'om', ' w', 'as', ' c', 'al', 'le', 'd\\n', 'du', 'mm', 'li', 'ng', ', ', 'an', 'd ', 'wa', 's ', 'de', 'sp', 'is', 'ed', ', ', 'mo', 'ck']\n",
      "\n",
      "Processing file stories\\051.txt\n",
      "Data size (Characters) (Document 50) 5608\n",
      "Sample string (Document 50) ['\\ta', 'll', 'er', 'le', 'ir', 'au', 'h\\n', '\\nt', 'he', 're', ' w', 'as', ' o', 'nc', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'a ', 'ki', 'ng', ' w', 'ho', ' h', 'ad', ' a', ' w', 'if', 'e ', 'wi', 'th', ' g', 'ol', 'de', 'n ', 'ha', 'ir', ',\\n', 'an', 'd ', 'sh', 'e ', 'wa', 's ', 'so', ' b', 'ea']\n",
      "\n",
      "Processing file stories\\052.txt\n",
      "Data size (Characters) (Document 51) 1287\n",
      "Sample string (Document 51) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'om', 'an', ' a', 'nd', ' h', 'er', ' d', 'au', 'gh', 'te', 'r ', 'wh', 'o ', 'li', 've', 'd ', 'in', ' a', '\\np', 're', 'tt', 'y ', 'ga', 'rd', 'en', ' w', 'it', 'h ', 'ca', 'bb', 'ag', 'es', '. ', ' a', 'nd', ' a', ' l', 'it', 'tl', 'e ', 'ha']\n",
      "\n",
      "Processing file stories\\053.txt\n",
      "Data size (Characters) (Document 52) 2841\n",
      "Sample string (Document 52) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', \"g'\", 's ', 'so', 'n ', 'wh', 'o ', 'ha', 'd ', 'a ', 'br', 'id', 'e ', 'wh', 'om', ' h', 'e ', 'lo', 've', 'd ', 've', 'ry', ' m', 'uc', 'h.', '\\na', 'nd', ' w', 'he', 'n ', 'he', ' w', 'as', ' s', 'it', 'ti', 'ng', ' b', 'es', 'id', 'e ']\n",
      "\n",
      "Processing file stories\\054.txt\n",
      "Data size (Characters) (Document 53) 1922\n",
      "Sample string (Document 53) ['ha', 'ns', ' w', 'is', 'he', 'd ', 'to', ' p', 'ut', ' h', 'is', ' s', 'on', ' t', 'o ', 'le', 'ar', 'n ', 'a ', 'tr', 'ad', 'e,', ' s', 'o ', 'he', ' w', 'en', 't ', 'in', 'to', ' t', 'he', '\\nc', 'hu', 'rc', 'h ', 'an', 'd ', 'pr', 'ay', 'ed', ' t', 'o ', 'ou', 'r ', 'lo', 'rd', ' g', 'od', ' t']\n",
      "\n",
      "Processing file stories\\055.txt\n",
      "Data size (Characters) (Document 54) 2573\n",
      "Sample string (Document 54) ['a ', 'fa', 'th', 'er', ' o', 'nc', 'e ', 'ca', 'll', 'ed', ' h', 'is', ' t', 'hr', 'ee', ' s', 'on', 's ', 'be', 'fo', 're', ' h', 'im', ', ', 'an', 'd ', 'he', ' g', 'av', 'e ', 'to', ' t', 'he', '\\nf', 'ir', 'st', ' a', ' c', 'oc', 'k,', ' t', 'o ', 'th', 'e ', 'se', 'co', 'nd', ' a', ' s', 'cy']\n",
      "\n",
      "Processing file stories\\056.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 55) 5285\n",
      "Sample string (Document 55) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' u', 'nd', 'er', 'st', 'oo', 'd ', 'al', 'l ', 'ki', 'nd', 's ', 'of', ' a', 'rt', 's.', '  ', 'he', ' s', 'er', 've', 'd ', 'in', '\\nw', 'ar', ', ', 'an', 'd ', 'be', 'ha', 've', 'd ', 'we', 'll', ' a', 'nd', ' b', 'ra', 've']\n",
      "\n",
      "Processing file stories\\057.txt\n",
      "Data size (Characters) (Document 56) 971\n",
      "Sample string (Document 56) ['th', 'e ', 'sh', 'e-', 'wo', 'lf', ' b', 'ro', 'ug', 'ht', ' i', 'nt', 'o ', 'th', 'e ', 'wo', 'rl', 'd ', 'a ', 'yo', 'un', 'g ', 'on', 'e,', ' a', 'nd', ' i', 'nv', 'it', 'ed', ' t', 'he', ' f', 'ox', '\\nt', 'o ', 'be', ' g', 'od', 'fa', 'th', 'er', '. ', ' a', 'ft', 'er', ' a', 'll', ', ', 'he']\n",
      "\n",
      "Processing file stories\\058.txt\n",
      "Data size (Characters) (Document 57) 4538\n",
      "Sample string (Document 57) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' q', 'ue', 'en', ' t', 'o ', 'wh', 'om', ' g', 'od', ' h', 'ad', ' g', 'iv', 'en', ' n', 'o ', 'ch', 'il', 'dr', 'en', '.\\n', 'ev', 'er', 'y ', 'mo', 'rn', 'in', 'g ', 'sh', 'e ', 'we', 'nt', ' i', 'nt', 'o ', 'th']\n",
      "\n",
      "Processing file stories\\059.txt\n",
      "Data size (Characters) (Document 58) 636\n",
      "Sample string (Document 58) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' v', 'er', 'y ', 'ol', 'd ', 'ma', 'n,', ' w', 'ho', 'se', ' e', 'ye', 's ', 'ha', 'd ', 'be', 'co', 'me', ' d', 'im', ', ', 'hi', 's ', 'ea', 'rs', '\\nd', 'ul', 'l ', 'of', ' h', 'ea', 'ri', 'ng', ', ', 'hi', 's ', 'kn', 'ee', 's ', 'tr', 'em', 'bl']\n",
      "\n",
      "Processing file stories\\060.txt\n",
      "Data size (Characters) (Document 59) 786\n",
      "Sample string (Document 59) ['a ', 'li', 'tt', 'le', ' b', 'ro', 'th', 'er', ' a', 'nd', ' s', 'is', 'te', 'r ', 'we', 're', ' o', 'nc', 'e ', 'pl', 'ay', 'in', 'g ', 'by', ' a', ' w', 'el', 'l,', ' a', 'nd', ' w', 'hi', 'le', '\\nt', 'he', 'y ', 'we', 're', ' t', 'hu', 's ', 'pl', 'ay', 'in', 'g,', ' t', 'he', 'y ', 'bo', 'th']\n",
      "\n",
      "Processing file stories\\061.txt\n",
      "Data size (Characters) (Document 60) 10687\n",
      "Sample string (Document 60) ['th', 'er', 'e ', 'wa', 's ', 'on', 'e ', 'up', 'on', ' a', ' t', 'im', 'e ', 'a ', 'gr', 'ea', 't ', 'wa', 'r,', ' a', 'nd', ' w', 'he', 'n ', 'it', ' c', 'am', 'e ', 'to', ' a', 'n ', 'en', 'd,', '\\nm', 'an', 'y ', 'so', 'ld', 'ie', 'rs', ' w', 'er', 'e ', 'di', 'sc', 'ha', 'rg', 'ed', '. ', ' t']\n",
      "\n",
      "Processing file stories\\062.txt\n",
      "Data size (Characters) (Document 61) 5105\n",
      "Sample string (Document 61) ['ha', 'ns', ' h', 'ad', ' s', 'er', 've', 'd ', 'hi', 's ', 'ma', 'st', 'er', ' f', 'or', ' s', 'ev', 'en', ' y', 'ea', 'rs', ', ', 'so', ' h', 'e ', 'sa', 'id', ' t', 'o ', 'hi', 'm,', '\\nm', 'as', 'te', 'r,', ' m', 'y ', 'ti', 'me', ' i', 's ', 'up', ', ', 'no', 'w ', 'i ', 'sh', 'ou', 'ld', ' b']\n",
      "\n",
      "Processing file stories\\063.txt\n",
      "Data size (Characters) (Document 62) 1127\n",
      "Sample string (Document 62) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' y', 'ou', 'ng', ' p', 'ea', 'sa', 'nt', ' n', 'am', 'ed', ' h', 'an', 's,', ' w', 'ho', 'se', ' u', 'nc', 'le', '\\nw', 'an', 'te', 'd ', 'to', ' f', 'in', 'd ', 'hi', 'm ', 'a ', 'ri', 'ch', ' w', 'if', 'e.', '  ']\n",
      "\n",
      "Processing file stories\\064.txt\n",
      "Data size (Characters) (Document 63) 4981\n",
      "Sample string (Document 63) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'ma', 'n ', 'an', 'd ', 'a ', 'po', 'or', ' w', 'om', 'an', ' w', 'ho', ' h', 'ad', ' n', 'ot', 'hi', 'ng', ' b', 'ut', ' a', '\\nl', 'it', 'tl', 'e ', 'co', 'tt', 'ag', 'e,', ' a', 'nd', ' w', 'ho', ' e', 'ar', 'ne', 'd ', 'th', 'ei']\n",
      "\n",
      "Processing file stories\\065.txt\n",
      "Data size (Characters) (Document 64) 6006\n",
      "Sample string (Document 64) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' m', 'an', ' w', 'ho', ' w', 'as', ' a', 'bo', 'ut', ' t', 'o ', 'se', 't ', 'ou', 't ', 'on', ' a', ' l', 'on', 'g\\n', 'jo', 'ur', 'ne', 'y,', ' a', 'nd', ' o', 'n ', 'pa', 'rt', 'in', 'g ', 'he', ' a', 'sk', 'ed']\n",
      "\n",
      "Processing file stories\\066.txt\n",
      "Data size (Characters) (Document 65) 5900\n",
      "Sample string (Document 65) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', 'n ', 'ol', 'd ', 'qu', 'ee', 'n ', 'wh', 'os', 'e ', 'hu', 'sb', 'an', 'd ', 'ha', 'd ', 'be', 'en', ' d', 'ea', 'd\\n', 'fo', 'r ', 'ma', 'ny', ' y', 'ea', 'rs', ', ', 'an', 'd ', 'sh', 'e ', 'ha', 'd ', 'a ', 'be']\n",
      "\n",
      "Processing file stories\\067.txt\n",
      "Data size (Characters) (Document 66) 7837\n",
      "Sample string (Document 66) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' c', 'ou', 'nt', 'ry', 'ma', 'n ', 'ha', 'd ', 'a ', 'so', 'n ', 'wh', 'o ', 'wa', 's ', 'as', ' b', 'ig', ' a', 's ', 'a ', 'th', 'um', 'b,', '\\na', 'nd', ' d', 'id', ' n', 'ot', ' b', 'ec', 'om', 'e ', 'an', 'y ', 'bi', 'gg', 'er', ', ', 'an']\n",
      "\n",
      "Processing file stories\\068.txt\n",
      "Data size (Characters) (Document 67) 4717\n",
      "Sample string (Document 67) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' r', 'ic', 'h ', 'ki', 'ng', ' w', 'ho', ' h', 'ad', ' t', 'hr', 'ee', ' d', 'au', 'gh', 'te', 'rs', ', ', 'wh', 'o\\n', 'da', 'il', 'y ', 'we', 'nt', ' t', 'o ', 'wa', 'lk', ' i', 'n ', 'th', 'e ', 'pa', 'la', 'ce']\n",
      "\n",
      "Processing file stories\\069.txt\n",
      "Data size (Characters) (Document 68) 6233\n",
      "Sample string (Document 68) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'ce', 'rt', 'ai', 'n ', 'me', 'rc', 'ha', 'nt', ' w', 'ho', ' h', 'ad', ' t', 'wo', ' c', 'hi', 'ld', 're', 'n,', ' a', ' b', 'oy', ' a', 'nd', ' a', ' g', 'ir', 'l,', '\\nt', 'he', 'y ', 'we', 're', ' b', 'ot', 'h ', 'yo', 'un', 'g,', ' a', 'nd', ' c', 'ou', 'ld']\n",
      "\n",
      "Processing file stories\\070.txt\n",
      "Data size (Characters) (Document 69) 5664\n",
      "Sample string (Document 69) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' q', 'ue', 'en', ' w', 'ho', ' h', 'ad', ' a', ' l', 'it', 'tl', 'e ', 'da', 'ug', 'ht', 'er', ' w', 'ho', '\\nw', 'as', ' s', 'ti', 'll', ' s', 'o ', 'yo', 'un', 'g ', 'th', 'at', ' s', 'he', ' h', 'ad', ' t', 'o ']\n",
      "\n",
      "Processing file stories\\071.txt\n",
      "Data size (Characters) (Document 70) 3569\n",
      "Sample string (Document 70) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'pe', 'as', 'an', 't ', 'wh', 'o ', 'ha', 'd ', 'no', ' l', 'an', 'd,', ' b', 'ut', ' o', 'nl', 'y ', 'a ', 'sm', 'al', 'l\\n', 'ho', 'us', 'e,', ' a', 'nd', ' o', 'ne', ' d', 'au', 'gh', 'te', 'r.', '  ', 'th', 'en', ' s', 'ai', 'd ']\n",
      "\n",
      "Processing file stories\\072.txt\n",
      "Data size (Characters) (Document 71) 3793\n",
      "Sample string (Document 71) ['ab', 'ou', 't ', 'a ', 'th', 'ou', 'sa', 'nd', ' o', 'r ', 'mo', 're', ' y', 'ea', 'rs', ' a', 'go', ', ', 'th', 'er', 'e ', 'we', 're', ' i', 'n ', 'th', 'is', '\\nc', 'ou', 'nt', 'ry', ' n', 'ot', 'hi', 'ng', ' b', 'ut', ' s', 'ma', 'll', ' k', 'in', 'gs', ', ', 'an', 'd ', 'on', 'e ', 'of', ' t']\n",
      "\n",
      "Processing file stories\\073.txt\n",
      "Data size (Characters) (Document 72) 5980\n",
      "Sample string (Document 72) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', 'g ', 'wh', 'o ', 'ha', 'd ', 'an', ' i', 'll', 'ne', 'ss', ', ', 'an', 'd ', 'no', ' o', 'ne', ' b', 'el', 'ie', 've', 'd ', 'th', 'at', ' h', 'e\\n', 'wo', 'ul', 'd ', 'co', 'me', ' o', 'ut', ' o', 'f ', 'it', ' w', 'it', 'h ', 'hi', 's ']\n",
      "\n",
      "Processing file stories\\074.txt\n",
      "Data size (Characters) (Document 73) 4518\n",
      "Sample string (Document 73) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'wo', 'od', 'cu', 'tt', 'er', ' w', 'ho', ' t', 'oi', 'le', 'd ', 'fr', 'om', ' e', 'ar', 'ly', '\\nm', 'or', 'ni', 'ng', ' t', 'il', 'l ', 'la', 'te', ' a', 't ', 'ni', 'gh', 't.', '  ', 'wh', 'en', ' a', 't ', 'la', 'st', ' h', 'e ']\n",
      "\n",
      "Processing file stories\\075.txt\n",
      "Data size (Characters) (Document 74) 3247\n",
      "Sample string (Document 74) ['a ', 'di', 'sc', 'ha', 'rg', 'ed', ' s', 'ol', 'di', 'er', ' h', 'ad', ' n', 'ot', 'hi', 'ng', ' t', 'o ', 'li', 've', ' o', 'n,', ' a', 'nd', ' d', 'id', ' n', 'ot', ' k', 'no', 'w ', 'ho', 'w ', 'to', '\\nm', 'ak', 'e ', 'hi', 's ', 'wa', 'y.', '  ', 'so', ' h', 'e ', 'we', 'nt', ' o', 'ut', ' i']\n",
      "\n",
      "Processing file stories\\076.txt\n",
      "Data size (Characters) (Document 75) 5130\n",
      "Sample string (Document 75) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' y', 'ou', 'ng', ' f', 'el', 'lo', 'w ', 'wh', 'o ', 'en', 'li', 'st', 'ed', ' a', 's ', 'a ', 'so', 'ld', 'ie', 'r,', ' c', 'on', 'du', 'ct', 'ed', '\\nh', 'im', 'se', 'lf', ' b', 'ra', 've', 'ly', ', ', 'an', 'd ', 'wa', 's ', 'al', 'wa', 'ys', ' t']\n",
      "\n",
      "Processing file stories\\077.txt\n",
      "Data size (Characters) (Document 76) 2401\n",
      "Sample string (Document 76) ['on', 'ce', ' i', 'n ', 'su', 'mm', 'er', '-t', 'im', 'e ', 'th', 'e ', 'be', 'ar', ' a', 'nd', ' t', 'he', ' w', 'ol', 'f ', 'we', 're', ' w', 'al', 'ki', 'ng', ' i', 'n ', 'th', 'e ', 'fo', 're', 'st', ',\\n', 'an', 'd ', 'th', 'e ', 'be', 'ar', ' h', 'ea', 'rd', ' a', ' b', 'ir', 'd ', 'si', 'ng']\n",
      "\n",
      "Processing file stories\\078.txt\n",
      "Data size (Characters) (Document 77) 624\n",
      "Sample string (Document 77) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'po', 'or', ' b', 'ut', ' g', 'oo', 'd ', 'li', 'tt', 'le', ' g', 'ir', 'l ', 'wh', 'o ', 'li', 've', 'd ', 'al', 'on', 'e ', 'wi', 'th', ' h', 'er', '\\nm', 'ot', 'he', 'r,', ' a', 'nd', ' t', 'he', 'y ', 'no', ' l', 'on', 'ge', 'r ', 'ha', 'd ', 'an', 'yt', 'hi']\n",
      "\n",
      "Processing file stories\\079.txt\n",
      "Data size (Characters) (Document 78) 3991\n",
      "Sample string (Document 78) ['on', 'e ', 'da', 'y ', 'a ', 'pe', 'as', 'an', 't ', 'to', 'ok', ' h', 'is', ' g', 'oo', 'd ', 'ha', 'ze', 'l-', 'st', 'ic', 'k ', 'ou', 't ', 'of', ' t', 'he', ' c', 'or', 'ne', 'r\\n', 'an', 'd ', 'sa', 'id', ' t', 'o ', 'hi', 's ', 'wi', 'fe', ', ', 'tr', 'in', 'a,', ' i', ' a', 'm ', 'go', 'in']\n",
      "\n",
      "Processing file stories\\080.txt\n",
      "Data size (Characters) (Document 79) 1426\n",
      "Sample string (Document 79) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' l', 'it', 'tl', 'e ', 'ch', 'il', 'd ', 'wh', 'os', 'e ', 'mo', 'th', 'er', ' g', 'av', 'e ', 'he', 'r ', 'ev', 'er', 'y\\n', 'af', 'te', 'rn', 'oo', 'n ', 'a ', 'sm', 'al', 'l ', 'bo', 'wl', ' o', 'f ', 'mi', 'lk', ' a', 'nd', ' b', 're', 'ad', ', ']\n",
      "\n",
      "Processing file stories\\081.txt\n",
      "Data size (Characters) (Document 80) 3574\n",
      "Sample string (Document 80) ['in', ' a', ' c', 'er', 'ta', 'in', ' m', 'il', 'l ', 'li', 've', 'd ', 'an', ' o', 'ld', ' m', 'il', 'le', 'r ', 'wh', 'o ', 'ha', 'd ', 'ne', 'it', 'he', 'r ', 'wi', 'fe', ' n', 'or', ' c', 'hi', 'ld', ',\\n', 'an', 'd ', 'th', 're', 'e ', 'ap', 'pr', 'en', 'ti', 'ce', 's ', 'se', 'rv', 'ed', ' u']\n",
      "\n",
      "Processing file stories\\082.txt\n",
      "Data size (Characters) (Document 81) 10822\n",
      "Sample string (Document 81) ['hi', 'll', ' a', 'nd', ' v', 'al', 'e ', 'do', ' n', 'ot', ' m', 'ee', 't,', ' b', 'ut', ' t', 'he', ' c', 'hi', 'ld', 're', 'n ', 'of', ' m', 'en', ' d', 'o,', ' g', 'oo', 'd ', 'an', 'd ', 'ba', 'd.', '\\ni', 'n ', 'th', 'is', ' w', 'ay', ' a', ' s', 'ho', 'em', 'ak', 'er', ' a', 'nd', ' a', ' t']\n",
      "\n",
      "Processing file stories\\083.txt\n",
      "Data size (Characters) (Document 82) 5480\n",
      "Sample string (Document 82) ['\\th', 'an', 's ', 'th', 'e ', 'he', 'dg', 'eh', 'og', '\\n\\n', 'th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' c', 'ou', 'nt', 'ry', ' m', 'an', ' w', 'ho', ' h', 'ad', ' m', 'on', 'ey', ' a', 'nd', ' l', 'an', 'd ', 'in', ' p', 'le', 'nt', 'y,', ' b', 'ut', '\\nh', 'ow', 'ev', 'er', ' r', 'ic', 'h ']\n",
      "\n",
      "Processing file stories\\084.txt\n",
      "Data size (Characters) (Document 83) 658\n",
      "Sample string (Document 83) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'ot', 'he', 'r ', 'wh', 'o ', 'ha', 'd ', 'a ', 'li', 'tt', 'le', ' b', 'oy', ' o', 'f ', 'se', 've', 'n ', 'ye', 'ar', 's ', 'ol', 'd,', ' w', 'ho', '\\nw', 'as', ' s', 'o ', 'ha', 'nd', 'so', 'me', ' a', 'nd', ' l', 'ov', 'ab', 'le', ' t', 'ha']\n",
      "\n",
      "Processing file stories\\085.txt\n",
      "Data size (Characters) (Document 84) 5989\n",
      "Sample string (Document 84) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' y', 'ou', 'ng', ' f', 'el', 'lo', 'w ', 'wh', 'o ', 'ha', 'd ', 'le', 'ar', 'nt', ' t', 'he', ' t', 'ra', 'de', ' o', 'f ', 'lo', 'ck', 'sm', 'it', 'h,', '\\na', 'nd', ' t', 'ol', 'd ', 'hi', 's ', 'fa', 'th', 'er', ' h', 'e ', 'wo', 'ul', 'd ', 'no']\n",
      "\n",
      "Processing file stories\\086.txt\n",
      "Data size (Characters) (Document 85) 8758\n",
      "Sample string (Document 85) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' k', 'in', 'g ', 'wh', 'o ', 'ha', 'd ', 'a ', 'li', 'tt', 'le', ' b', 'oy', ' i', 'n ', 'wh', 'os', 'e ', 'st', 'ar', 's\\n', 'it', ' h', 'ad', ' b', 'ee', 'n ', 'fo', 're', 'to', 'ld', ' t', 'ha', 't ', 'he', ' s']\n",
      "\n",
      "Processing file stories\\087.txt\n",
      "Data size (Characters) (Document 86) 3109\n",
      "Sample string (Document 86) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' a', ' p', 'ri', 'nc', 'es', 's ', 'wh', 'o ', 'wa', 's ', 'ex', 'tr', 'em', 'el', 'y ', 'pr', 'ou', 'd.', ' i', 'f ', 'a\\n', 'wo', 'oe', 'r ', 'ca', 'me', ' s', 'he', ' g', 'av', 'e ', 'hi', 'm ', 'so', 'me', ' r', 'id']\n",
      "\n",
      "Processing file stories\\088.txt\n",
      "Data size (Characters) (Document 87) 1365\n",
      "Sample string (Document 87) ['a ', 'ta', 'il', 'or', \"'s\", ' a', 'pp', 're', 'nt', 'ic', 'e ', 'wa', 's ', 'tr', 'av', 'el', 'in', 'g ', 'ab', 'ou', 't ', 'th', 'e ', 'wo', 'rl', 'd ', 'in', ' s', 'ea', 'rc', 'h ', 'of', '\\nw', 'or', 'k,', ' a', 'nd', ' a', 't ', 'on', 'e ', 'ti', 'me', ' h', 'e ', 'co', 'ul', 'd ', 'fi', 'nd']\n",
      "\n",
      "Processing file stories\\089.txt\n",
      "Data size (Characters) (Document 88) 4538\n",
      "Sample string (Document 88) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' o', 'n ', 'a ', 'ti', 'me', ' a', ' s', 'ol', 'di', 'er', ' w', 'ho', ' f', 'or', ' m', 'an', 'y ', 'ye', 'ar', 's ', 'ha', 'd ', 'se', 'rv', 'ed', ' t', 'he', '\\nk', 'in', 'g ', 'fa', 'it', 'hf', 'ul', 'ly', ', ', 'bu', 't ', 'wh', 'en', ' t', 'he', ' w']\n",
      "\n",
      "Processing file stories\\090.txt\n",
      "Data size (Characters) (Document 89) 345\n",
      "Sample string (Document 89) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' t', 'he', 're', ' w', 'as', ' a', ' c', 'hi', 'ld', ' w', 'ho', ' w', 'as', ' w', 'il', 'lf', 'ul', ', ', 'an', 'd ', 'wo', 'ul', 'd ', 'no', 't ', 'do', '\\nw', 'ha', 't ', 'he', 'r ', 'mo', 'th', 'er', ' w', 'is', 'he', 'd.', '  ', 'fo', 'r ', 'th']\n",
      "\n",
      "Processing file stories\\091.txt\n",
      "Data size (Characters) (Document 90) 5460\n",
      "Sample string (Document 90) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' k', 'in', \"g'\", 's ', 'so', 'n,', ' w', 'ho', ' w', 'as', ' n', 'o ', 'lo', 'ng', 'er', ' c', 'on', 'te', 'nt', ' t', 'o ', 'st', 'ay', ' a', 't\\n', 'ho', 'me', ' i', 'n ', 'hi', 's ', 'fa', 'th', 'er', \"'s\", ' h', 'ou', 'se', ', ', 'an', 'd ', 'as']\n",
      "\n",
      "Processing file stories\\092.txt\n",
      "Data size (Characters) (Document 91) 6854\n",
      "Sample string (Document 91) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' y', 'ou', 'ng', ' h', 'un', 'ts', 'ma', 'n ', 'wh', 'o ', 'we', 'nt', ' i', 'nt', 'o ', 'th', 'e ', 'fo', 're', 'st', ' t', 'o ', 'li', 'e ', 'in', '\\nw', 'ai', 't.', '  ', 'he', ' h', 'ad', ' a', ' f', 're', 'sh', ' a', 'nd', ' j', 'oy', 'ou', 's ']\n",
      "\n",
      "Processing file stories\\093.txt\n",
      "Data size (Characters) (Document 92) 2314\n",
      "Sample string (Document 92) ['a ', 'po', 'or', ' s', 'er', 'va', 'nt', '-g', 'ir', 'l ', 'wa', 's ', 'on', 'ce', ' t', 'ra', 've', 'li', 'ng', ' w', 'it', 'h ', 'th', 'e ', 'fa', 'mi', 'ly', ' w', 'it', 'h ', 'wh', 'ic', 'h ', 'sh', 'e\\n', 'wa', 's ', 'in', ' s', 'er', 'vi', 'ce', ', ', 'th', 'ro', 'ug', 'h ', 'a ', 'gr', 'ea']\n",
      "\n",
      "Processing file stories\\094.txt\n",
      "Data size (Characters) (Document 93) 1706\n",
      "Sample string (Document 93) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' m', 'an', ' w', 'ho', ' h', 'ad', ' t', 'hr', 'ee', ' s', 'on', 's,', ' a', 'nd', ' n', 'ot', 'hi', 'ng', ' e', 'ls', 'e ', 'in', ' t', 'he', '\\nw', 'or', 'ld', ' b', 'ut', ' t', 'he', ' h', 'ou', 'se', ' i', 'n ', 'wh', 'ic', 'h ', 'he', ' l', 'iv']\n",
      "\n",
      "Processing file stories\\095.txt\n",
      "Data size (Characters) (Document 94) 3229\n",
      "Sample string (Document 94) ['th', 'er', 'e ', 'wa', 's ', 'a ', 'gr', 'ea', 't ', 'wa', 'r,', ' a', 'nd', ' t', 'he', ' k', 'in', 'g ', 'ha', 'd ', 'ma', 'ny', ' s', 'ol', 'di', 'er', 's,', ' b', 'ut', ' g', 'av', 'e ', 'th', 'em', '\\ns', 'ma', 'll', ' p', 'ay', ', ', 'so', ' s', 'ma', 'll', ' t', 'ha', 't ', 'th', 'ey', ' c']\n",
      "\n",
      "Processing file stories\\096.txt\n",
      "Data size (Characters) (Document 95) 4954\n",
      "Sample string (Document 95) ['on', 'ce', ' u', 'po', 'n ', 'a ', 'ti', 'me', ' l', 'iv', 'ed', ' a', ' m', 'an', ' a', 'nd', ' a', ' w', 'om', 'an', ' w', 'ho', ' s', 'o ', 'lo', 'ng', ' a', 's ', 'th', 'ey', ' w', 'er', 'e\\n', 'ri', 'ch', ' h', 'ad', ' n', 'o ', 'ch', 'il', 'dr', 'en', ', ', 'bu', 't ', 'wh', 'en', ' t', 'he']\n",
      "\n",
      "Processing file stories\\097.txt\n",
      "Data size (Characters) (Document 96) 5732\n",
      "Sample string (Document 96) ['in', ' t', 'he', ' d', 'ay', 's ', 'wh', 'en', ' w', 'is', 'hi', 'ng', ' w', 'as', ' s', 'ti', 'll', ' o', 'f ', 'so', 'me', ' u', 'se', ', ', 'a ', 'ki', 'ng', \"'s\", ' s', 'on', ' w', 'as', '\\nb', 'ew', 'it', 'ch', 'ed', ' b', 'y ', 'an', ' o', 'ld', ' w', 'it', 'ch', ', ', 'an', 'd ', 'sh', 'ut']\n",
      "\n",
      "Processing file stories\\098.txt\n",
      "Data size (Characters) (Document 97) 4334\n",
      "Sample string (Document 97) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' p', 'oo', 'r ', 'ma', 'n ', 'wh', 'o ', 'ha', 'd ', 'fo', 'ur', ' s', 'on', 's,', ' a', 'nd', ' w', 'he', 'n ', 'th', 'ey', ' w', 'er', 'e ', 'gr', 'ow', 'n\\n', 'up', ', ', 'he', ' s', 'ai', 'd ', 'to', ' t', 'he', 'm,', ' \"', 'my', ' d', 'ea', 'r ']\n",
      "\n",
      "Processing file stories\\099.txt\n",
      "Data size (Characters) (Document 98) 7090\n",
      "Sample string (Document 98) ['th', 'er', 'e ', 'wa', 's ', 'on', 'ce', ' a', ' w', 'om', 'an', ' w', 'ho', ' h', 'ad', ' t', 'hr', 'ee', ' d', 'au', 'gh', 'te', 'rs', ', ', 'th', 'e ', 'el', 'de', 'st', ' o', 'f ', 'wh', 'om', '\\nw', 'as', ' c', 'al', 'le', 'd ', 'on', 'e-', 'ey', 'e,', ' b', 'ec', 'au', 'se', ' s', 'he', ' h']\n",
      "\n",
      "Processing file stories\\100.txt\n",
      "Data size (Characters) (Document 99) 1007\n",
      "Sample string (Document 99) ['\"g', 'oo', 'd-', 'da', 'y,', ' f', 'at', 'he', 'r ', 'ho', 'll', 'en', 'th', 'e.', '\" ', '\"m', 'an', 'y ', 'th', 'an', 'ks', ', ', 'pi', 'f-', 'pa', 'f-', 'po', 'lt', 'ri', 'e.', '\" ', '\"m', 'ay', ' i', '\\nb', 'e ', 'al', 'lo', 'we', 'd ', 'to', ' h', 'av', 'e ', 'yo', 'ur', ' d', 'au', 'gh', 'te']\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "  \n",
    "  with open(filename) as f:\n",
    "    data = tf.compat.as_str(f.read())\n",
    "    # make all the words lower case\n",
    "    data = data.lower()\n",
    "    data = list(data)\n",
    "  return data\n",
    "\n",
    "documents = []\n",
    "global documents\n",
    "for i in range(num_files):    \n",
    "    print('\\nProcessing file %s'%os.path.join(dir_name,filenames[i]))\n",
    "    chars = read_data(os.path.join(dir_name,filenames[i]))\n",
    "    \n",
    "    # Break the data into bigrams\n",
    "    two_grams = [''.join(chars[ch_i:ch_i+2]) for ch_i in range(0,len(chars)-2,2)]\n",
    "    # Create a list of lists with bigrams\n",
    "    documents.append(two_grams)\n",
    "    print('Data size (Characters) (Document %d) %d' %(i,len(two_grams)))\n",
    "    print('Sample string (Document %d) %s'%(i,two_grams[:50]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the Dictionaries (Bigrams)\n",
    "Builds the following. To understand each of these elements, let us also assume the text \"I like to go to school\"\n",
    "\n",
    "* `dictionary`: maps a string word to an ID (e.g. {I:0, like:1, to:2, go:3, school:4})\n",
    "* `reverse_dictionary`: maps an ID to a string word (e.g. {0:I, 1:like, 2:to, 3:go, 4:school}\n",
    "* `count`: List of list of (word, frequency) elements (e.g. [(I,1),(like,1),(to,2),(go,1),(school,1)]\n",
    "* `data` : Contain the string of text we read, where string words are replaced with word IDs (e.g. [0, 1, 2, 3, 2, 4])\n",
    "\n",
    "It also introduces an additional special token `UNK` to denote rare words to are too rare to make use of."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "449177 Characters found.\n",
      "Most common words (+UNK) [('e ', 15229), ('he', 15164), (' t', 13443), ('th', 13076), ('d ', 10687)]\n",
      "Least common words (+UNK) [('; ', 1), ('z ', 1), (\"'v\", 1), ('tz', 1), (\"a'\", 1), ('\\ts', 1), ('hc', 1), ('xq', 1), ('zi', 1), ('i?', 1), ('pw', 1), ('zo', 1), ('hd', 1), ('\\tc', 1), ('dh', 1)]\n",
      "Sample data [15, 28, 86, 23, 3, 95, 74, 11, 2, 16]\n",
      "Sample data [22, 156, 25, 37, 83, 185, 43, 9, 90, 19]\n",
      "Vocabulary:  544\n"
     ]
    }
   ],
   "source": [
    "def build_dataset(documents):\n",
    "    chars = []\n",
    "    # This is going to be a list of lists\n",
    "    # Where the outer list denote each document\n",
    "    # and the inner lists denote words in a given document\n",
    "    data_list = []\n",
    "  \n",
    "    for d in documents:\n",
    "        chars.extend(d)\n",
    "    print('%d Characters found.'%len(chars))\n",
    "    count = []\n",
    "    # Get the bigram sorted by their frequency (Highest comes first)\n",
    "    count.extend(collections.Counter(chars).most_common())\n",
    "    \n",
    "    # Create an ID for each bigram by giving the current length of the dictionary\n",
    "    # And adding that item to the dictionary\n",
    "    # Start with 'UNK' that is assigned to too rare words\n",
    "    dictionary = dict({'UNK':0})\n",
    "    for char, c in count:\n",
    "        # Only add a bigram to dictionary if its frequency is more than 10\n",
    "        if c > 10:\n",
    "            dictionary[char] = len(dictionary)    \n",
    "    \n",
    "    unk_count = 0\n",
    "    # Traverse through all the text we have\n",
    "    # to replace each string word with the ID of the word\n",
    "    for d in documents:\n",
    "        data = list()\n",
    "        for char in d:\n",
    "            # If word is in the dictionary use the word ID,\n",
    "            # else use the ID of the special token \"UNK\"\n",
    "            if char in dictionary:\n",
    "                index = dictionary[char]        \n",
    "            else:\n",
    "                index = dictionary['UNK']\n",
    "                unk_count += 1\n",
    "            data.append(index)\n",
    "            \n",
    "        data_list.append(data)\n",
    "        \n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n",
    "    return data_list, count, dictionary, reverse_dictionary\n",
    "\n",
    "global data_list, count, dictionary, reverse_dictionary,vocabulary_size\n",
    "\n",
    "# Print some statistics about data\n",
    "data_list, count, dictionary, reverse_dictionary = build_dataset(documents)\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Least common words (+UNK)', count[-15:])\n",
    "print('Sample data', data_list[0][:10])\n",
    "print('Sample data', data_list[1][:10])\n",
    "print('Vocabulary: ',len(dictionary))\n",
    "vocabulary_size = len(dictionary)\n",
    "del documents  # To reduce memory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Batches of Data\n",
    "The following object generates a batch of data which will be used to train the LSTM. More specifically the generator breaks a given sequence of words into `batch_size` segments. We also maintain a cursor for each segment. So whenever we create a batch of data, we sample one item from each segment and update the cursor of each segment. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Unrolled index 0\n",
      "\tInputs:\n",
      "\te  (1), \tki (131), \t d (48), \t w (11), \tbe (70), \n",
      "\tOutput:\n",
      "\tli (98), \tng (33), \tau (195), \ter (14), \tau (195), \n",
      "\n",
      "Unrolled index 1\n",
      "\tInputs:\n",
      "\tli (98), \tng (33), \tau (195), \ter (14), \tau (195), \n",
      "\tOutput:\n",
      "\tve (41), \t\n",
      "w (169), \tgh (106), \te  (1), \tti (112), \n",
      "\n",
      "Unrolled index 2\n",
      "\tInputs:\n",
      "\tve (41), \t\n",
      "w (169), \tgh (106), \te  (1), \tti (112), \n",
      "\tOutput:\n",
      "\td  (5), \tho (62), \tte (61), \tal (84), \tfu (229), \n",
      "\n",
      "Unrolled index 3\n",
      "\tInputs:\n",
      "\td  (5), \tho (62), \tte (61), \tal (84), \tfu (229), \n",
      "\tOutput:\n",
      "\ta  (83), \tse (58), \trs (137), \tl  (57), \tl, (257), \n",
      "\n",
      "Unrolled index 4\n",
      "\tInputs:\n",
      "\ta  (83), \tse (58), \trs (137), \tl  (57), \tbe (70), \n",
      "\tOutput:\n",
      "\tki (131), \t d (48), \t w (11), \tbe (70), \tau (195), "
     ]
    }
   ],
   "source": [
    "class DataGeneratorOHE(object):\n",
    "    \n",
    "    def __init__(self,text,batch_size,num_unroll):\n",
    "        # Text where a bigram is denoted by its ID\n",
    "        self._text = text\n",
    "        # Number of bigrams in the text\n",
    "        self._text_size = len(self._text)\n",
    "        # Number of datapoints in a batch of data\n",
    "        self._batch_size = batch_size\n",
    "        # Num unroll is the number of steps we unroll the RNN in a single training step\n",
    "        # This relates to the truncated backpropagation we discuss in Chapter 6 text\n",
    "        self._num_unroll = num_unroll\n",
    "        # We break the text in to several segments and the batch of data is sampled by\n",
    "        # sampling a single item from a single segment\n",
    "        self._segments = self._text_size//self._batch_size\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "    def next_batch(self):\n",
    "        '''\n",
    "        Generates a single batch of data\n",
    "        '''\n",
    "        # Train inputs (one-hot-encoded) and train outputs (one-hot-encoded)\n",
    "        batch_data = np.zeros((self._batch_size,vocabulary_size),dtype=np.float32)\n",
    "        batch_labels = np.zeros((self._batch_size,vocabulary_size),dtype=np.float32)\n",
    "        \n",
    "        # Fill in the batch datapoint by datapoint\n",
    "        for b in range(self._batch_size):\n",
    "            # If the cursor of a given segment exceeds the segment length\n",
    "            # we reset the cursor back to the beginning of that segment\n",
    "            if self._cursor[b]+1>=self._text_size:\n",
    "                self._cursor[b] = b * self._segments\n",
    "            \n",
    "            # Add the text at the cursor as the input\n",
    "            batch_data[b,self._text[self._cursor[b]]] = 1.0\n",
    "            # Add the preceding bigram as the label to be predicted\n",
    "            batch_labels[b,self._text[self._cursor[b]+1]]= 1.0                       \n",
    "            # Update the cursor\n",
    "            self._cursor[b] = (self._cursor[b]+1)%self._text_size\n",
    "                    \n",
    "        return batch_data,batch_labels\n",
    "        \n",
    "    def unroll_batches(self):\n",
    "        '''\n",
    "        This produces a list of num_unroll batches\n",
    "        as required by a single step of training of the RNN\n",
    "        '''\n",
    "        unroll_data,unroll_labels = [],[]\n",
    "        for ui in range(self._num_unroll):\n",
    "            data, labels = self.next_batch()            \n",
    "            unroll_data.append(data)\n",
    "            unroll_labels.append(labels)\n",
    "        \n",
    "        return unroll_data, unroll_labels\n",
    "    \n",
    "    def reset_indices(self):\n",
    "        '''\n",
    "        Used to reset all the cursors if needed\n",
    "        '''\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "# Running a tiny set to see if things are correct\n",
    "dg = DataGeneratorOHE(data_list[0][25:50],5,5)\n",
    "u_data, u_labels = dg.unroll_batches()\n",
    "\n",
    "# Iterate through each data batch in the unrolled set of batches\n",
    "for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):   \n",
    "    print('\\n\\nUnrolled index %d'%ui)\n",
    "    dat_ind = np.argmax(dat,axis=1)\n",
    "    lbl_ind = np.argmax(lbl,axis=1)\n",
    "    print('\\tInputs:')\n",
    "    for sing_dat in dat_ind:\n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_dat],sing_dat),end=\", \")\n",
    "    print('\\n\\tOutput:')\n",
    "    for sing_lbl in lbl_ind:        \n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_lbl],sing_lbl),end=\", \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the LSTM\n",
    "\n",
    "This is a standard LSTM. The LSTM has 5 main components.\n",
    "* Cell state\n",
    "* Hidden state\n",
    "* Input gate\n",
    "* Forget gate\n",
    "* Output gate\n",
    "\n",
    "Each gate has three sets of weights (1 set for the current input, 1 set for the previous hidden state and 1 bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining hyperparameters\n",
    "\n",
    "Here we define several hyperparameters and are very similar to the ones we defined in Chapter 6. However additionally we use dropout; a technique that helps to avoid overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Number of neurons in the hidden state variables\n",
    "num_nodes = 128\n",
    "\n",
    "# Number of data points in a batch we process\n",
    "batch_size = 64\n",
    "\n",
    "# Number of time steps we unroll for during optimization\n",
    "num_unrollings = 50\n",
    "\n",
    "dropout = 0.0 # We use dropout\n",
    "\n",
    "# Use this in the CSV filename when saving\n",
    "# when using dropout\n",
    "filename_extension = ''\n",
    "if dropout>0.0:\n",
    "    filename_extension = '_dropout'\n",
    "    \n",
    "filename_to_save = 'lstm'+filename_extension+'.csv' # use to save perplexity values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Inputs and Outputs\n",
    "\n",
    "In the code we define two different types of inputs. \n",
    "* Training inputs (The stories we downloaded) (batch_size > 1 with unrolling)\n",
    "* Validation inputs (An unseen validation dataset) (bach_size =1, no unrolling)\n",
    "* Test input (New story we are going to generate) (batch_size=1, no unrolling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Training Input data.\n",
    "train_inputs, train_labels = [],[]\n",
    "\n",
    "# Defining unrolled training inputs\n",
    "for ui in range(num_unrollings):\n",
    "    train_inputs.append(tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size],name='train_inputs_%d'%ui))\n",
    "    train_labels.append(tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size], name = 'train_labels_%d'%ui))\n",
    "\n",
    "# Validation data placeholders\n",
    "valid_inputs = tf.placeholder(tf.float32, shape=[1,vocabulary_size],name='valid_inputs')\n",
    "valid_labels = tf.placeholder(tf.float32, shape=[1,vocabulary_size], name = 'valid_labels')\n",
    "# Text generation: batch 1, no unrolling.\n",
    "test_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size], name = 'test_input')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Model Parameters\n",
    "\n",
    "Now we define model parameters. Compared to RNNs, LSTMs have a large number of parameters. Each gate (input, forget, memory and output) has three different sets of parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Input gate (i_t) - How much memory to write to cell state\n",
    "# Connects the current input to the input gate\n",
    "ix = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the input gate\n",
    "im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the input gate\n",
    "ib = tf.Variable(tf.random_uniform([1, num_nodes],-0.02, 0.02))\n",
    "\n",
    "# Forget gate (f_t) - How much memory to discard from cell state\n",
    "# Connects the current input to the forget gate\n",
    "fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the forget gate\n",
    "fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the forget gate\n",
    "fb = tf.Variable(tf.random_uniform([1, num_nodes],-0.02, 0.02))\n",
    "\n",
    "# Candidate value (c~_t) - Used to compute the current cell state\n",
    "# Connects the current input to the candidate\n",
    "cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the candidate\n",
    "cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the candidate\n",
    "cb = tf.Variable(tf.random_uniform([1, num_nodes],-0.02,0.02))\n",
    "\n",
    "# Output gate (o_t) - How much memory to output from the cell state\n",
    "# Connects the current input to the output gate\n",
    "ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the output gate\n",
    "om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the output gate\n",
    "ob = tf.Variable(tf.random_uniform([1, num_nodes],-0.02,0.02))\n",
    "\n",
    "\n",
    "# Softmax Classifier weights and biases.\n",
    "w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], stddev=0.02))\n",
    "b = tf.Variable(tf.random_uniform([vocabulary_size],-0.02,0.02))\n",
    "\n",
    "# Variables saving state across unrollings.\n",
    "# Hidden state\n",
    "saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False, name='train_hidden')\n",
    "# Cell state\n",
    "saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False, name='train_cell')\n",
    "\n",
    "# Same variables for validation phase\n",
    "saved_valid_output = tf.Variable(tf.zeros([1, num_nodes]),trainable=False, name='valid_hidden')\n",
    "saved_valid_state = tf.Variable(tf.zeros([1, num_nodes]),trainable=False, name='valid_cell')\n",
    "\n",
    "# Same variables for testing phase\n",
    "saved_test_output = tf.Variable(tf.zeros([1, num_nodes]),trainable=False, name='test_hidden')\n",
    "saved_test_state = tf.Variable(tf.zeros([1, num_nodes]),trainable=False, name='test_cell')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining LSTM computations\n",
    "Here first we define the LSTM cell computations as a consice function. Then we use this function to define training and test-time inference logic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Definition of the cell computation.\n",
    "def lstm_cell(i, o, state):\n",
    "    \"\"\"Create an LSTM cell\"\"\"\n",
    "    input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n",
    "    forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n",
    "    update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "    state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "    output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n",
    "    return output_gate * tf.tanh(state), state\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# =========================================================\n",
    "#Training related inference logic\n",
    "\n",
    "# Keeps the calculated state outputs in all the unrollings\n",
    "# Used to calculate loss\n",
    "outputs = list()\n",
    "\n",
    "# These two python variables are iteratively updated\n",
    "# at each step of unrolling\n",
    "output = saved_output\n",
    "state = saved_state\n",
    "\n",
    "# Compute the hidden state (output) and cell state (state)\n",
    "# recursively for all the steps in unrolling\n",
    "for i in train_inputs:\n",
    "    output, state = lstm_cell(i, output, state)\n",
    "    output = tf.nn.dropout(output,keep_prob=1.0-dropout)\n",
    "    # Append each computed output value\n",
    "    outputs.append(output)\n",
    "\n",
    "# calculate the score values\n",
    "logits = tf.matmul(tf.concat(axis=0, values=outputs), w) + b\n",
    "    \n",
    "# Compute predictions.\n",
    "train_prediction = tf.nn.softmax(logits)\n",
    "\n",
    "# Compute training perplexity\n",
    "train_perplexity_without_exp = tf.reduce_sum(tf.concat(train_labels,0)*-tf.log(tf.concat(train_prediction,0)+1e-10))/(num_unrollings*batch_size)\n",
    "\n",
    "# ========================================================================\n",
    "# Validation phase related inference logic\n",
    "\n",
    "# Compute the LSTM cell output for validation data\n",
    "valid_output, valid_state = lstm_cell(\n",
    "    valid_inputs, saved_valid_output, saved_valid_state)\n",
    "# Compute the logits\n",
    "valid_logits = tf.nn.xw_plus_b(valid_output, w, b)\n",
    "\n",
    "# Make sure that the state variables are updated\n",
    "# before moving on to the next iteration of generation\n",
    "with tf.control_dependencies([saved_valid_output.assign(valid_output),\n",
    "                            saved_valid_state.assign(valid_state)]):\n",
    "    valid_prediction = tf.nn.softmax(valid_logits)\n",
    "\n",
    "# Compute validation perplexity\n",
    "valid_perplexity_without_exp = tf.reduce_sum(valid_labels*-tf.log(valid_prediction+1e-10))\n",
    "\n",
    "# ========================================================================\n",
    "# Testing phase related inference logic\n",
    "\n",
    "# Compute the LSTM cell output for testing data\n",
    "test_output, test_state = lstm_cell(\n",
    "test_input, saved_test_output, saved_test_state)\n",
    "\n",
    "# Make sure that the state variables are updated\n",
    "# before moving on to the next iteration of generation\n",
    "with tf.control_dependencies([saved_test_output.assign(test_output),\n",
    "                            saved_test_state.assign(test_state)]):\n",
    "    test_prediction = tf.nn.softmax(tf.nn.xw_plus_b(test_output, w, b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LSTM Loss\n",
    "We calculate the training loss of the LSTM here. It's a typical cross entropy loss calculated over all the scores we obtained for training data (`loss`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Before calcualting the training loss,\n",
    "# save the hidden state and the cell state to\n",
    "# their respective TensorFlow variables\n",
    "with tf.control_dependencies([saved_output.assign(output),\n",
    "                            saved_state.assign(state)]):\n",
    "\n",
    "    # Calculate the training loss by\n",
    "    # concatenating the results from all the unrolled time steps\n",
    "    loss = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits_v2(\n",
    "        logits=logits, labels=tf.concat(axis=0, values=train_labels)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Learning Rate and the Optimizer with Gradient Clipping\n",
    "Here we define the learning rate and the optimizer we're going to use. We will be using the Adam optimizer as it is one of the best optimizers out there. Furthermore we use gradient clipping to prevent any gradient explosions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# learning rate decay\n",
    "gstep = tf.Variable(0,trainable=False,name='global_step')\n",
    "\n",
    "# Running this operation will cause the value of gstep\n",
    "# to increase, while in turn reducing the learning rate\n",
    "inc_gstep = tf.assign(gstep, gstep+1)\n",
    "\n",
    "# Decays learning rate everytime the gstep increases\n",
    "tf_learning_rate = tf.train.exponential_decay(0.001,gstep,decay_steps=1, decay_rate=0.5)\n",
    "\n",
    "# Adam Optimizer. And gradient clipping.\n",
    "optimizer = tf.train.AdamOptimizer(tf_learning_rate)\n",
    "gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "# Clipping gradients\n",
    "gradients, _ = tf.clip_by_global_norm(gradients, 5.0)\n",
    "\n",
    "optimizer = optimizer.apply_gradients(\n",
    "    zip(gradients, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resetting Operations for Resetting Hidden States\n",
    "Sometimes the state variable needs to be reset (e.g. when starting predictions at a beginning of a new epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reset train state\n",
    "reset_train_state = tf.group(tf.assign(saved_state, tf.zeros([batch_size, num_nodes])),\n",
    "                          tf.assign(saved_output, tf.zeros([batch_size, num_nodes])))\n",
    "\n",
    "# Reset valid state\n",
    "reset_valid_state = tf.group(tf.assign(saved_valid_state, tf.zeros([1, num_nodes])),\n",
    "                          tf.assign(saved_valid_output, tf.zeros([1, num_nodes])))\n",
    "\n",
    "# Reset test state\n",
    "reset_test_state = tf.group(\n",
    "    saved_test_output.assign(tf.random_normal([1, num_nodes],stddev=0.05)),\n",
    "    saved_test_state.assign(tf.random_normal([1, num_nodes],stddev=0.05)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Greedy Sampling to Break the Repetition\n",
    "Here we write some simple logic to break the repetition in text. Specifically instead of always getting the word that gave this highest prediction probability, we sample randomly where the probability of being selected given by their prediction probabilities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample(distribution):\n",
    "  '''Greedy Sampling\n",
    "  We pick the three best predictions given by the LSTM and sample\n",
    "  one of them with very high probability of picking the best one'''\n",
    "\n",
    "  best_inds = np.argsort(distribution)[-3:]\n",
    "  best_probs = distribution[best_inds]/np.sum(distribution[best_inds])\n",
    "  best_idx = np.random.choice(best_inds,p=best_probs)\n",
    "  return best_idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the LSTM to Generate Text\n",
    "\n",
    "Here we train the LSTM on the available data and generate text using the trained LSTM for several steps. From each document we extract text for `steps_per_document` steps to train the LSTM on. We also report the train perplexity at the end of each step. Finally we test the LSTM by asking it to generate some new text starting from a randomly picked bigram."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning rate Decay Logic\n",
    "\n",
    "Here we define the logic to decrease learning rate whenever the validation perplexity does not decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning rate decay related\n",
    "# If valid perpelxity does not decrease\n",
    "# continuously for this many epochs\n",
    "# decrease the learning rate\n",
    "decay_threshold = 5\n",
    "# Keep counting perplexity increases\n",
    "decay_count = 0\n",
    "\n",
    "min_perplexity = 1e10\n",
    "\n",
    "# Learning rate decay logic\n",
    "def decay_learning_rate(session, v_perplexity):\n",
    "  global decay_threshold, decay_count, min_perplexity  \n",
    "  # Decay learning rate\n",
    "  if v_perplexity < min_perplexity:\n",
    "    decay_count = 0\n",
    "    min_perplexity= v_perplexity\n",
    "  else:\n",
    "    decay_count += 1\n",
    "\n",
    "  if decay_count >= decay_threshold:\n",
    "    print('\\t Reducing learning rate')\n",
    "    decay_count = 0\n",
    "    session.run(inc_gstep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running Training, Validation and Generation\n",
    "\n",
    "We traing the LSTM on existing training data, check the validaiton perplexity on an unseen chunk of text and generate a fresh segment of text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized Global Variables \n",
      "Training (Step: 0) (16).(96).(81).(36).(54).(58).(57).(20).(12).(82).\n",
      "Average loss at step 1: 4.277536\n",
      "\tPerplexity at step 1: 72.062685\n",
      "\n",
      "Valid Perplexity: 61.50\n",
      "\n",
      "Generated Text after epoch 0 ... \n",
      "======================== New text Segment ==========================\n",
      "\t shut and a she was he was to the hed, the king he was to the was he was to was with he he king, and to he had to his his shome and he king's he king, and the king's he wad to when he was he was father, and the was to the hergethe was to the was was sone had then the was her, and had to that the king, and had her her he he was the was to to had to whe her, and had then his he was to her she her, and haver he with him to his she was with ther to the haves to he was the king had shat he hed, and that with he hed had with the had the was father, and\n",
      "had to her, and had he with he here, and him, and the her to he was to the cone he his sith her to was with the was\n",
      "the king, and had to was sone the was to the was his hand that him, and the king, and the was the was\n",
      "to was she he had when the whoe he hergeand, the he was to her then him to he was to her, and\n",
      "he hed him, and that with him to he he her, and hat he whe was was with the king, and the had the king, and he hed her shad the king, and \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 1) (14).(79).(8).(9).(69).(30).(43).(73).(40).(4).\n",
      "Average loss at step 2: 3.156932\n",
      "\tPerplexity at step 2: 23.498395\n",
      "\n",
      "Valid Perplexity: 39.69\n",
      "\n",
      "Generated Text after epoch 1 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  huse of had bried and then to his bearded he have him, and that she had to beard to him, and he her to been she and then the rest faithful john ang the said the king, and what his beaut his beaut and he saw he the queen said the\n",
      "king of the king then and took the king with his the sained the here, that was said to the queen to that is with the king begain and will began ing to his had became that had then ing beare bed to he was of her then the king had saway then and tooked he her hento the king will faithful john and to her to the king to the king faines took the knows to the king with then then and took the know the king had said it the king wife had said to the shave to one to the king to her to hall to that him, and he her the king became his but then the to his prest and he said 'that it thin to that forthis said the would\n",
      "hallen to the king threes of heads, and took the king to shall the saine to the king was said to that he him, and he said the king thrus feard, and to the shent\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 2) (74).(20).(84).(4).(64).(96).(16).(39).(61).(8).\n",
      "Average loss at step 3: 2.437171\n",
      "\tPerplexity at step 3: 11.440626\n",
      "\n",
      "Valid Perplexity: 57.38\n",
      "\n",
      "Generated Text after epoch 2 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  care for it be and the inst the let home to a frol down.  and dest, and will take her great up of the a serroord, and when\n",
      "one thene tountil leave inced withingly that he weresteaut out in her into the came of her, and was so to a chancessed as your everyery.  then thereat hear of i seemisented to while, so you beaught for me, but one ronts towear to her and becaus that your sto his every for some to a forrow,\n",
      "       rapunzel, with the hear, a sickel knew he had down a giratered to that he were towy to\n",
      "the had not in wre thand he wife the restroamed himself sto said of a get of\n",
      "thater, and then he was so her the wored to you.  you will him all see said, will do, her rapunzel, rapunzel is thing stme, and delivery beaut to her you will necond will to your hair, she with the stone to her hap\n",
      "it the inst, buve me roame, and in the stone if loss side her, and the\n",
      "paired at eared mery offered that you with a came to her, her let your, tower he with one into her, and a pare of\n",
      "chances, and at \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 3) (30).(60).(29).(82).(88).(94).(75).(66).(95).(11).\n",
      "Average loss at step 4: 2.384119\n",
      "\tPerplexity at step 4: 10.849499\n",
      "\n",
      "Valid Perplexity: 34.29\n",
      "\n",
      "Generated Text after epoch 3 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  and gived together.  then she cried off, and walked to on to dom.  so them, and threw it on their, that she shall be and little hanself.  then said the children had nonever, after the old wanted to getch they well to the fastened himself on hed stleng that, she was said, but it the witch, and grate up the cameles into the fores, howeread neen she was to they hover\n",
      "pickly as and the stils, and they were into the witcUNKs house.  they gretel said, and the duck, that as with how, the witch,\n",
      "and look that is the woman, when the little and each other, and threw them they had not so far\n",
      "hansel to her.  then she cried, and the door, but she would not is hansel hansel, and the still houshe bried, and saw that it the\n",
      "moretes and there was in to began to go on about the children reads ing to father hansel, and said hansel, they were themselves little sinto them.  there was lefted to that is will to ther.  there them, and wanted the witcUNKs of to her, is hall by and the mast me, and which here had \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 4) (58).(75).(49).(94).(55).(56).(11).(27).(38).(95).\n",
      "Average loss at step 5: 1.730926\n",
      "\tPerplexity at step 5: 5.645881\n",
      "\n",
      "Valid Perplexity: 49.90\n",
      "\n",
      "Generated Text after epoch 4 ... \n",
      "======================== New text Segment ==========================\n",
      "\t ches?\" \"oh,\" said, \"i under their trip, that he had to get i are, and on there, and she came on a whit i was cringrythe that let off him hease he was to get up and flowed, and when all i was aware i and sallove, and laugh he thirder, and now i am to you toge, who had nothre, the thath st tow.  and the princess noth i could not get it\n",
      "on, and that it would not\n",
      "stay.  and the king beautiful had been still of ferdinand the\n",
      "faithful.  once its brour into her to lo little she to that she had stought of them by, who make he have again for him that it which had about to him, and the giants and the birds what he have you\n",
      "learnt that?\" \"oh, years, his seven you with me to the faithful was celeping his faithful was on it, and the ship, were shall be then at to said to his head, but do the man whit he had better the king, she called toget do you at ferdinand\n",
      "ferdinand the faithful, again\n",
      "at the other, and what had bringhted himself and said heared the wholers of the king was bright, whip him. and t\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 5) (44).(80).(59).(29).(40).(41).(76).(73).(50).(52).\n",
      "Average loss at step 6: 1.909839\n",
      "\tPerplexity at step 6: 6.752005\n",
      "\n",
      "Valid Perplexity: 46.21\n",
      "\n",
      "Generated Text after epoch 5 ... \n",
      "======================== New text Segment ==========================\n",
      "\t l becaust bride, and when\n",
      "the king's necaughter, and he had to project.  when the\n",
      "morning inlihe chim, and they came to the king's she had happed to his heart was learned to who her were going to be put to the started the kingdom, for he had a wife\n",
      "already, and someone who had found an old key did never yet the king said to hame, they are that the king's have bride, and we who appened the king, and\n",
      "saw  to be put to the test,\n",
      "and had the spinning just\n",
      "as here and that when\n",
      "the lioned the which had that they said, and at they have st this world.  the king to liever they the king's daughter wentinto thuntsmen, and they servant to the trow.\n",
      "\n",
      "the the pins who have to me areplen, and what it the king's who had the red they walk and dowse.  they were goint to themsetered the lion.\n",
      "\n",
      "the staken into have some, it was take them, and she stay from the king's daughter thather an eacked\n",
      "the king said, the king, i will someon, anked to the tworls was\n",
      "life, and the spirst them, that her long had happe\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 6) (38).(35).(10).(97).(45).(76).(20).(73).(27).(68).\n",
      "Average loss at step 7: 1.739238\n",
      "\tPerplexity at step 7: 5.693001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Valid Perplexity: 44.98\n",
      "\n",
      "Generated Text after epoch 6 ... \n",
      "======================== New text Segment ==========================\n",
      "\t ers, and to the warder, in the colem, the power, and\n",
      "the maiden, whom you bearn to she was a great aston, and she went to his hands lying of a sleep.  and my will fell it in for of and\n",
      "the whold to his father, and then into word, but he had not to believere asled into her and all where pass, and he alone was asled ment other.  then he went away himself, and then he to tword herself but the father, i have been on him, out of the foit on the thought behome, but he would no look of the golden what i will you must set him their trance.  he\n",
      "was tere, into him fore wedding to the have the facele, and\n",
      "her the\n",
      "pere in not true.  then he last shalt was their father, where the morestrew that his have of mage, and as he had go aw her what i divid then, the\n",
      "betray,\n",
      "then his herdsomer cank one of think, and shall said it will you.\n",
      "\n",
      "when the money of me, trey was as she again, head to the poweither word, they he were to go oner me, in it reat for your head, but help tried the wolf, and said, no, we as\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 7) (71).(9).(25).(2).(31).(35).(58).(54).(18).(65).\n",
      "Average loss at step 8: 1.608271\n",
      "\tPerplexity at step 8: 4.994170\n",
      "\n",
      "Valid Perplexity: 47.25\n",
      "\n",
      "Generated Text after epoch 7 ... \n",
      "======================== New text Segment ==========================\n",
      "\t unting-maid he pas is it,\n",
      "and took into the\n",
      "came a beathroshedged to palace happed on the place of answenting, heard that him alhat thuse draw the renmostreet in the pliees, i have have not to the cat, and her to the glaster and asked, and took the saicid with as thone of good to ans wind the other, was trude, and there was to take, and askng for she still said,\n",
      "          but, she have darrested the chile, and said, i had now where came, and said, god grant that a beautifully when he son that he was sing, it wanting has before,\n",
      "whated invited aft a hall nother to lord that the king's daughter and ence what which hes had berrors, and and be a people waided to him my her, and ask when said, and the wing wing have a kumblince had from a loss and brought in the peace-stere, and it, and hearder my will, which was not in again, soon youther, and the maid such n, how in the serles's bread, and pring when you carried, and you aresd and why way,\n",
      "          and said, whon\n",
      "as i so en took you these \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 8) (40).(58).(51).(42).(99).(94).(83).(76).(62).(9).\n",
      "Average loss at step 9: 1.214943\n",
      "\tPerplexity at step 9: 3.370102\n",
      "\n",
      "Valid Perplexity: 44.30\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 8 ... \n",
      "======================== New text Segment ==========================\n",
      "\t for a\n",
      "vil for a liarted wife so they and sawne of she beautiful great how ister.  the hors again but when he tool, and did not groom came to and it was stooking to got cate to anim that which she stood off her spother so much day no one of it, and said, the scullion could grong cried the brothers with\n",
      "such she was sorrow into the king, who had a violen fat pall out in to tain, and the old woman\n",
      "could not\n",
      "the baUNK\n",
      "\n",
      "who she was as grage come into the king, in a little longed the man's day that\n",
      "the bror and\n",
      "been hered to see the\n",
      "little men had good-for in, she came in the quiek of the ground.\n",
      "\n",
      "when as her head\n",
      "taid the king to the king, who\n",
      "came with his sword answered the queen, nuleased the crack, the little men.  then i slepped\n",
      "how answered it had been their put on the dead, but he saw that, and wherew a little little one out, and is that doer\n",
      "the little men.  yes are is so the coubated some of word,\n",
      "what doeve herself by the\n",
      "man, it down to a viisted answered on the was and mothouse\n",
      "sto\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 9) (77).(76).(17).(21).(49).(94).(12).(15).(46).(81).\n",
      "Average loss at step 10: 2.071144\n",
      "\tPerplexity at step 10: 7.933896\n",
      "\n",
      "Valid Perplexity: 22.54\n",
      "\n",
      "Generated Text after epoch 9 ... \n",
      "======================== New text Segment ==========================\n",
      "\t to be the maiden, and said, her took his stood, but the king had to see on the stood that the king had became to the broad wittle bade and said, \"if they was also that it was one or them round hisseather was not in the windo for you, and said, i has becan ame have tren her many thing or the lighted to his was one on the forth of with the fountay one and home to the little had to be shold he to the tailor, and were golden his the could no one\n",
      "you have said to the time have you be and sand on his house.  no of why cook him with the huntsman as he radeneart the stood on the for the tailor it to his wif in he was aloned our have but to the husband to said the tailor it his bride alred her was the little pear which the king to the little pardon.  then he had came wing, to be shappered in her wanted to look that the king's son was the shollow and he carried the for st the same.  he took him out the lived her\n",
      "beggs a little of the little was said, \"in the shoes, and they were the water, and the\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 10) (80).(43).(71).(2).(53).(94).(15).(10).(87).(92).\n",
      "Average loss at step 11: 1.807553\n",
      "\tPerplexity at step 11: 6.095511\n",
      "\n",
      "Valid Perplexity: 38.90\n",
      "\n",
      "Generated Text after epoch 10 ... \n",
      "======================== New text Segment ==========================\n",
      "\t ck, she saw if his horse, and were tesen in therried, and had glas stood the ring was wind a great herself and his\n",
      "kingdom, for the window it, and then him. hor her her, and went to her the bird, and the hare dide alonly, and\n",
      "house, and likivishere girl, and a brought his hand, and said, no moster and stood it, and lived all youd for sed the bird, when the little had jumpsecrets they were free into the wate and there, whom when he had to saw it, and\n",
      "hold shoode of rind to she round\n",
      "herself at him.  after to likewise she from, and who had not restrang it was not began forth a sing that was a toge it, and said, a now, the shorld will dove and\n",
      "wane should not lit anoth, and said, and hous also not look again, and said -  and maused them fortune was from his wank it away with\n",
      "her, and liked with at he thouse. then she also it.  but the girl the youth said he will sat down, the horte a lovely flood, for it.  and\n",
      "wast to the this, man he and were to me to me a bright white you will my four cou\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 11) (22).(39).(56).(90).(5).(40).(72).(67).(79).(42).\n",
      "Average loss at step 12: 2.113701\n",
      "\tPerplexity at step 12: 8.278822\n",
      "\n",
      "Valid Perplexity: 28.01\n",
      "\n",
      "Generated Text after epoch 11 ... \n",
      "======================== New text Segment ==========================\n",
      "\t d which\n",
      "had been stopped, and said the knep the\n",
      "child and him a mealon, but of the hands and which went hall the girl hat he was shall it was\n",
      "here, some me stilled.\n",
      "\n",
      "then when were beautiful mane that in went belp the cooks, and the but never meat was the king, and against the cound as to be forting the had not come out to put it down to a sittle longer of the horse and said the you the king and had greatesered and goom for some me to the\n",
      "clothing before the mant head of your bode i come by the charesand have at the kick, and when the people with\n",
      "it, and his for much replied the husban and all deavers he told\n",
      "her jew he was still of hearth could not do now, but the\n",
      "old greated to have that and said, \"i will that he were di and then at the took his command.  it said, \"i have no some was a little have take the paddled his cameed with her house to day but a longers had to be seizen a long, but they come together, but it was sat and his daughter and heard to him in that was and i had they ha\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 12) (50).(34).(36).(90).(77).(27).(4).(97).(33).(1).\n",
      "Average loss at step 13: 2.062049\n",
      "\tPerplexity at step 13: 7.862059\n",
      "\n",
      "Valid Perplexity: 27.40\n",
      "\n",
      "Generated Text after epoch 12 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t her was morned the beaut looked for nothing, i had liking only, and the king she once heard.  if you will set first one.' the countrer, and stooy one of the hand he had deasun the forbidlen, and and looked out wanted the two lasked\n",
      "for the forbidden maided the bride when the great some quite child but the stairly down the king, and her so the pigeoped forth, and she have a bove farthers her to the stove, the will go out to be said 'i will gave her to his daughter home all open the demnvered she forbide and her in his dearessed and there pasked for her elf, and aterm, and the master deaven sonswed and drounded, and the sheast to be as loth arenswer, i will set her to dring of you.  when the king care not to\n",
      "restled to the queen as the king's son, so the people\n",
      "mother stood oness am loudly ther that his\n",
      "causeting, the child was than the cound, and he the desries of it, the father his dareaver head to her, and she looked in a girst about her from her a fearme, i will she seen in your sides \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 13) (74).(87).(3).(63).(92).(89).(31).(98).(96).(88).\n",
      "Average loss at step 14: 1.794309\n",
      "\tPerplexity at step 14: 6.015317\n",
      "\n",
      "Valid Perplexity: 28.33\n",
      "\n",
      "Generated Text after epoch 13 ... \n",
      "======================== New text Segment ==========================\n",
      "\t nce who brought his street form in his daughter.\" when you said, \"what, the wild have you, the devil had to he had, and the that with his life.  \"i want and i will.\" own to the escolUNKere.  and the manner was all three do threet, and thressed they were no servants again, and that they all that no see then she lived all a great forest, and a said that the mannikin was pincing-shudged.\n",
      "\n",
      "he went away to\n",
      "the king and asked with the death.  who have you are not.  when the king's son were to his fore of who she had put in the pocket and cried them, he had end stones sood,\n",
      "and she dred\n",
      "the lighted that she was a hearth to him hat for a pare when said the king, and hold as soon as a bride with sto the this daughter.  \"who make my night in histle.  what it is night, he was and the soldier that his hand it in the whole had seeping, he had\n",
      "bride away fraused him.  when this she said, heard that she said, \"what she would holf had became whith him, and the king was therewards, which he had some by wi\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 14) (49).(65).(59).(50).(23).(82).(74).(29).(31).(1).\n",
      "Average loss at step 15: 1.697946\n",
      "\tPerplexity at step 15: 5.462715\n",
      "\n",
      "Valid Perplexity: 33.81\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 14 ... \n",
      "======================== New text Segment ==========================\n",
      "\t tree, and when the youngest, and she remained thus she carried the forest her too there end all the robber and\n",
      "shout ill your mother, and when she got to her, and sting the forr was stratched out the violend just as head.  she had deen\n",
      "devoice.  next mordor, and answer, whold say to them time of the this what lefore should have set the\n",
      "finger, and that whered\n",
      "a long to drest.  there as at her the\n",
      "herself seen the the soldier then she said the down on you, and walling you\n",
      "a long time already\n",
      "you have your been child came back to him, with the devirgin about to the boot our could.  but the forthe when in the stay light but a brother, and\n",
      "the girl looked for the forbidden down to her, the first by the son, and in a great ledge, and spropered to the her, and still commanted the forth it was a butcher was a likes, and came back your the queen forth it, and the third, she came to him, and when the stood behind the maiden, and the quite botty this she did another and saw the secrad and left on \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 15) (47).(20).(53).(50).(35).(40).(22).(15).(67).(7).\n",
      "Average loss at step 16: 2.181118\n",
      "\tPerplexity at step 16: 8.856205\n",
      "\n",
      "Valid Perplexity: 22.34\n",
      "\n",
      "Generated Text after epoch 15 ... \n",
      "======================== New text Segment ==========================\n",
      "\t erriend to the water, and there to the daughter was to for the of churse fore in the backed the maiden went into the drancUNKs of the window and was into the fire.' then the king fortune to the glace, and seen now the bride was before to the tell into the forest was in the wife again.\" then that they had eld not not do that, he was she led down.  the king was she saw the deather and come in a stove, the down, and said, for he had seen again.  then they were to the bottle, inhe was she had seen.  then the king for the false him to kill your bross, and\n",
      "was that he answered through the plaked her word with her, and said, i will severead, and saw the wither whole kest her, but the king and led their was golden were it, but on the door\n",
      "began, whan the worder was tell into the forest was in the way.\n",
      "the maiden was she had something the devil had behind the room, and that would not days the dow one answered more to the tore next morning to becat let her behind the door spind to the dear not bega\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 16) (38).(84).(46).(98).(69).(83).(86).(9).(0).(1).\n",
      "Average loss at step 17: 2.270382\n",
      "\tPerplexity at step 17: 9.683102\n",
      "\n",
      "Valid Perplexity: 26.08\n",
      "\n",
      "Generated Text after epoch 16 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  the stood on the forbtill and so heard it, and was a little thres abore for the queen rejoiced to take me into the virgin lighted the table he was terrified the first, and the maid was full and she saw the wood her had toged the forbidden door, and the third time this warming that the king's son was sorrow it, and cried,\n",
      "UNKhis dear from the virgin mountilt, and after then he came back again, but that she went to piece if the first and\n",
      "threw her hair the brideest growted that the manchanged to den about of the\n",
      "mother was about her she couse to be and for the the stone of the hands and began on your with she saw the door.  he could not seep, and hew her, so she put the did night heard that when he cooked to him her in to hill, and the virgin mare descened until\n",
      "that the marriage were only and look and when the handkold, and cut the tole world.  what she had son had solden fruit, in the virge together and was taken again brought a mornight, she was a hunger, and ate said the door and she w\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 17) (52).(9).(10).(42).(83).(94).(40).(96).(27).(41).\n",
      "Average loss at step 18: 1.783976\n",
      "\tPerplexity at step 18: 5.953482\n",
      "\n",
      "Valid Perplexity: 21.55\n",
      "\n",
      "Generated Text after epoch 17 ... \n",
      "======================== New text Segment ==========================\n",
      "\t then hand, when they said, \"what dood he saw at the king's son was at laster, and then the people saw the maiden who was ang fell into his seater and went out of his\n",
      "more, and the king's son was and said, it was full of pold of look of his good day, that it was with his was and had been in again, so they came to the found to the cook of his peater, when they were to his answered, and they have nothing out of the face.  there, and the mannot were frout of the stood answered, and the maiden was secry, but as it\n",
      "as lenger.  and said, \"i will that i cannot be as she asked her.  he was dearefore him to his kingdom.\n",
      "\n",
      "but the wollor,\n",
      "and when she wanted to see his father's\n",
      "great are at her the old made stood they shall be beggained the worldier good little day will her, should not been the\n",
      "forest as still\n",
      "mock, they shall not to long to and the found had took her, and took\n",
      "the would had the king's\n",
      "daughter, for the third time.  then she was\n",
      "set up for and so fall the rest, so not in it was side\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 18) (31).(21).(88).(47).(23).(46).(63).(37).(11).(64).\n",
      "Average loss at step 19: 2.142808\n",
      "\tPerplexity at step 19: 8.523338\n",
      "\n",
      "Valid Perplexity: 19.20\n",
      "\n",
      "Generated Text after epoch 18 ... \n",
      "======================== New text Segment ==========================\n",
      "\t r\n",
      "which it had no earth.  they came to his forest before, and the dragon on when they were she lived it them to be put it to asleep into the world.  when she was no longer\n",
      "soon but the was abothered them the third time, the king see the door in the could, and the maiden were great wanted, and they shall in the hairl said, and there\n",
      "which the lived down.  then the princess, threw the servant was six on she, and then their friest and that she would be had been comed.  then she was a great joy, that is answered, and when he was so me in it, and said, i have earened him and said to her beggar-rost then when they hans.  which her and their dear nothing, and when it had seen.  when she seated\n",
      "the little dause that he came with their fell down and went into the world, that she came the bride, and three down to see to her, but their prounced to them, and after their parrified once more their wedding it there, and they had been that the blace kinsed him to the little dress forth he was full on fo\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 19) (6).(55).(46).(70).(48).(21).(77).(3).(47).(41).\n",
      "Average loss at step 20: 2.083418\n",
      "\tPerplexity at step 20: 8.031878\n",
      "\n",
      "Valid Perplexity: 24.21\n",
      "\n",
      "Generated Text after epoch 19 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  put the winred fountil where it was the ring when they had to let hom an her said, where on the little poor you, and went witch she saw that she would be prouncenty.  the little have on in the wolfer were fround the ring, and they went away for the\n",
      "ofven, he was not for and the peasant was good but she could.  after her.\n",
      "\n",
      "they were one, and then she stovens and the window and was\n",
      "not into the two brothers reached with the satisfied.  then they were conted in and still of roast mouse, and then they were opened the fire it and went up, had everything which he had, and there came to her, but was to set the stones, and led herself to which and said, she went to a stoght, and she answered, then the child came the princess her had to each off the ate to seep, and said, \"i will be told to go to look an and where the given me and speak and said, stook their down what he was not on the broken the wedding-soon.  the king said, the lake, the humber's daughter, and\n",
      "went to she remained alsoother, a\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 20) (1).(62).(11).(48).(70).(8).(9).(98).(35).(79).\n",
      "Average loss at step 21: 1.954084\n",
      "\tPerplexity at step 21: 7.057451\n",
      "\n",
      "Valid Perplexity: 27.91\n",
      "\n",
      "Generated Text after epoch 20 ... \n",
      "======================== New text Segment ==========================\n",
      "\t y little sisters, and wanted to cooking, but the children, and then he crept her the children the stable, and wept up\n",
      "to fries,\n",
      "ither child alone it, and said, the doess little shome by the\n",
      "more fortunated herself into her, and the stoor womes and little she cried, and said, i\n",
      "took her came back,\n",
      "                                 iUNK\n",
      "the grown\n",
      "which is the child came of its what i he stayed through the palace, whereon their dear of the places which the chill beautiful climon one\n",
      "she chair again again, but\n",
      "her sat was against the mock,\n",
      "and now whater he hairst child, said the woman was letted hand is it took at her the wollor, showed a little golden secondind of the pridered the chailses came in the coor, and said, \"i have not be and was standing in the wands the king came to the paddown.  she heard that looked once more the children, and they saw the word said, she heard the paddock with his find on the\n",
      "mother said, \"if i were furt of treasure,\n",
      "and the king came to the paddock with it her\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 21) (46).(13).(1).(81).(85).(76).(61).(99).(98).(54).\n",
      "Average loss at step 22: 2.232047\n",
      "\tPerplexity at step 22: 9.318925\n",
      "\n",
      "Valid Perplexity: 25.68\n",
      "\n",
      "Generated Text after epoch 21 ... \n",
      "======================== New text Segment ==========================\n",
      "\t imment out as well darried, and to be found the room, hedges that the twite answered, the king was so dear mound, which was none day again.  the girl have no more the palace, and in the stookis the king seen heard a nevery stones their death.\n",
      "\n",
      "at little made withen, who had\n",
      "been a treast catring to see his eyes, and\n",
      "three heard was house, and they hans we thone she had to will nothing thirst so that was she and there stood was at up, and at last at last it care again to stand until the king's son, and they were everythre were her pawered the young perhaps the stones was after her to\n",
      "do wands their frounlord that, where in the\n",
      "middle of the might, was oness which was serrined to such her took the cat, when she was sather to deathing for tend, and the third said he, but they hans, and sat down together to do the roung when the king's son had found her lead\n",
      "by their children away,\n",
      "but the\n",
      "bround that the thoes would cat oness, the might\n",
      "in the readow, and the better.\" she gaveed the bround,\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 22) (29).(71).(80).(88).(98).(83).(44).(41).(48).(47).\n",
      "Average loss at step 23: 1.682953\n",
      "\tPerplexity at step 23: 5.381422\n",
      "\n",
      "Valid Perplexity: 29.55\n",
      "\n",
      "Generated Text after epoch 22 ... \n",
      "======================== New text Segment ==========================\n",
      "\t  and said, and thither the old woman had that\n",
      "her so much a magnificent dicele herself in the water in the eation had down beautiful wentione stood threw his life, how have a tured of the\n",
      "three maidenin the children, and beathe\n",
      "morned them came out of her sing, and said, my daughter herself to death the sing which the other in the room and someth was was at last that the child came out one king said to bed their father and princess, and at ugh the other together, and came her, and brother the\n",
      "water the king.\n",
      "\n",
      "so that\n",
      "it be brose ove themselves been that he was at that the two othered the one woman.  that was so the ross thoughter to him.  then the woman, and thought them the seven at last their chunger was than they wept with the wait, and wept with them and they came to the king.\n",
      "\n",
      "the shoes the king when i cried the roast meath.  she came to them, and threethre, the princess was twoy than you mound when the longer old wertanding on the bird the fing of the were stood three-eyes, and the\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 23) (9).(16).(99).(89).(86).(75).(17).(37).(19).(1).\n",
      "Average loss at step 24: 1.838739\n",
      "\tPerplexity at step 24: 6.288606\n",
      "\n",
      "Valid Perplexity: 24.51\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 23 ... \n",
      "======================== New text Segment ==========================\n",
      "\t   the other all that she had seen forth, and took her to anything, and sted the litten senter the horse of the woosed and carried the fish, and hered she went up, and she sprang up again.\n",
      "when the cook gave her and looked at her and before her, and speopled out the king and took her tood the the store is her in the could, but she was the first was began to said to take the house in the child, and her and said, what does my horse was she had son had put in the housand held, and i witchere in two\n",
      "little sate the little might be she thought her and her fair in a huntsman and said, take thered, she\n",
      "her with me\n",
      "nice, the king took it was still as the viller, and held her was alones by her arm, and the her time of brother, but it was all the youngers he could he lay dear shown her in the appear, and they had been cutter.  then she was delivered the third there, and the queen had to sisters to the grandmhat was still down to her branch are get the fires and so she\n",
      "could he took the queen had th\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 24) (75).(54).(8).(37).(30).(91).(41).(80).(32).(60).\n",
      "Average loss at step 25: 2.344544\n",
      "\tPerplexity at step 25: 10.428514\n",
      "\n",
      "Valid Perplexity: 18.26\n",
      "\n",
      "Generated Text after epoch 24 ... \n",
      "======================== New text Segment ==========================\n",
      "\t t that she was into a hand, and\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the king's done the wants of the would as he done once had nothing out of his and was alone of them the seven it, the man had brought her hight beautiful may led see the man half all young that you have necone of the standing to be your with them, and\n",
      "said, where once, and the king's son witting, the king's son hoof a peakend and drink to her, the maiden to the saw this was of the man better.\" then, he was\n",
      "standing.  the servants to heaven, into that is began to see, and then the king said, i will to her, and drano, and the queen was, and there, and fell, and when they came the world, and when she had nothing three with him and the king's sonceived himself, but the\n",
      "old woman, said there, they have that, and there is the second a fought her.  but when she saw hould mark the read, and they and cracked it, and they were in a moster an enter to him, what he was dear wife and the\n",
      "home that he would be on to the kiss, what he had not knocked at his forthe to him\n",
      "====================================================================\n",
      "\n",
      "Training (Step: 25) (94).(80).(93).(43).(15).(28).(78).(91).(88).(44).\n",
      "Average loss at step 26: 2.263537\n",
      "\tPerplexity at step 26: 9.617045\n",
      "\n",
      "Valid Perplexity: 21.96\n",
      "\n",
      "Generated Text after epoch 25 ... \n",
      "======================== New text Segment ==========================\n",
      "\t ing with a house, and when they were all tere her your tailoUNK  then he was not soon and she strange and said, \"sill, what is not one of the horse and they were there, answered himself with them.\n",
      "the king and the prided, so it we diatannone, who has a land, and he was she\n",
      "came to his beart, and that the king's son as she had coned him, and then she would not so asked, councillick to him all all that the hang one sitting to the servant back from her his brought in the\n",
      "morning\n",
      "saw the devil, he said, i will stainly to heald to the king was the king, but in the could let then in in and wanted to does.  the said to the seven he said, went his bridescaint, the maid to where the root.  who know you can dest day but have the other had fallen, who was do\n",
      "sing to see if the white world.\n",
      "\n",
      "after that he had man onde himses, and then he could not so is to became not the sold, i will also strike this, and then she could not be\n",
      "together, and\n",
      "that the devil had son, and said, i will sought, the whole s\n",
      "====================================================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Some hyperparameters needed for the training process\n",
    "\n",
    "num_steps = 26\n",
    "steps_per_document = 100\n",
    "valid_summary = 1\n",
    "train_doc_count = 100\n",
    "docs_per_step = 10\n",
    "\n",
    "\n",
    "# Capture the behavior of train perplexity over time\n",
    "train_perplexity_ot = []\n",
    "valid_perplexity_ot = []\n",
    "\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "# Initializing variables\n",
    "tf.global_variables_initializer().run()\n",
    "print('Initialized Global Variables ')\n",
    "\n",
    "average_loss = 0 # Calculates the average loss ever few steps\n",
    "\n",
    "# We use the first 10 documents that has \n",
    "# more than 10*steps_per_document bigrams for creating the validation dataset\n",
    "\n",
    "# Identify the first 10 documents following the above condition\n",
    "long_doc_ids = []\n",
    "for di in range(num_files):\n",
    "  if len(data_list[di])>10*steps_per_document:\n",
    "    long_doc_ids.append(di)\n",
    "  if len(long_doc_ids)==10:\n",
    "    break\n",
    "    \n",
    "# Generating validation data\n",
    "data_gens = []\n",
    "valid_gens = []\n",
    "for fi in range(num_files):\n",
    "  # Get all the bigrams if the document id is not in the validation document ids\n",
    "  if fi not in long_doc_ids:\n",
    "    data_gens.append(DataGeneratorOHE(data_list[fi],batch_size,num_unrollings))\n",
    "  # if the document is in the validation doc ids, only get up to the \n",
    "  # last steps_per_document bigrams and use the last steps_per_document bigrams as validation data\n",
    "  else:\n",
    "    data_gens.append(DataGeneratorOHE(data_list[fi][:-steps_per_document],batch_size,num_unrollings))\n",
    "    # Defining the validation data generator\n",
    "    valid_gens.append(DataGeneratorOHE(data_list[fi][-steps_per_document:],1,1))\n",
    "\n",
    "\n",
    "feed_dict = {}\n",
    "for step in range(num_steps):\n",
    "    \n",
    "    print('Training (Step: %d)'%step,end=' ')\n",
    "    for di in np.random.permutation(train_doc_count)[:docs_per_step]:            \n",
    "        doc_perplexity = 0\n",
    "        for doc_step_id in range(steps_per_document):\n",
    "            \n",
    "            # Get a set of unrolled batches\n",
    "            u_data, u_labels = data_gens[di].unroll_batches()\n",
    "            \n",
    "            # Populate the feed dict by using each of the data batches\n",
    "            # present in the unrolled data\n",
    "            for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):            \n",
    "                feed_dict[train_inputs[ui]] = dat\n",
    "                feed_dict[train_labels[ui]] = lbl\n",
    "            \n",
    "            # Running the TensorFlow operations\n",
    "            _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n",
    "                                                       feed_dict=feed_dict)\n",
    "            # Update doc_perpelxity variable\n",
    "            doc_perplexity += step_perplexity \n",
    "            \n",
    "            # Update the average_loss variable\n",
    "            average_loss += step_perplexity \n",
    "            \n",
    "        # Show the printing progress <train_doc_id_1>.<train_doc_id_2>. ...\n",
    "        print('(%d).'%di,end='') \n",
    "    \n",
    "        # resetting hidden state after processing a single document\n",
    "        # It's still questionable if this adds value in terms of learning\n",
    "        # One one hand it's intuitive to reset the state when learning a new document\n",
    "        # On the other hand this approach creates a bias for the state to be zero\n",
    "        # We encourage the reader to investigate further the effect of resetting the state\n",
    "        #session.run(reset_train_state) # resetting hidden state for each document\n",
    "    print('')\n",
    "    \n",
    "    # Generate new samples\n",
    "    if (step+1) % valid_summary == 0:\n",
    "      \n",
    "      # Compute average loss\n",
    "      average_loss = average_loss / (valid_summary*docs_per_step*steps_per_document)\n",
    "      \n",
    "      # Print losses\n",
    "      print('Average loss at step %d: %f' % (step+1, average_loss))\n",
    "      print('\\tPerplexity at step %d: %f' %(step+1, np.exp(average_loss)))\n",
    "      train_perplexity_ot.append(np.exp(average_loss))\n",
    "        \n",
    "      average_loss = 0 # reset loss\n",
    "      valid_loss = 0 # reset loss\n",
    "        \n",
    "      # calculate valid perplexity\n",
    "      for v_doc_id in range(10):\n",
    "          # Remember we process things as bigrams\n",
    "          # So need to divide by 2\n",
    "          for v_step in range(steps_per_document//2):\n",
    "            uvalid_data,uvalid_labels = valid_gens[v_doc_id].unroll_batches()        \n",
    "\n",
    "            # Run validation phase related TensorFlow operations       \n",
    "            v_perp = session.run(\n",
    "                valid_perplexity_without_exp,\n",
    "                feed_dict = {valid_inputs:uvalid_data[0],valid_labels: uvalid_labels[0]}\n",
    "            )\n",
    "\n",
    "            valid_loss += v_perp\n",
    "            \n",
    "          session.run(reset_valid_state)\n",
    "      \n",
    "          # Reset validation data generator cursor\n",
    "          valid_gens[v_doc_id].reset_indices()      \n",
    "    \n",
    "      print()\n",
    "      v_perplexity = np.exp(valid_loss/(steps_per_document*10.0//2))\n",
    "      print(\"Valid Perplexity: %.2f\\n\"%v_perplexity)\n",
    "      valid_perplexity_ot.append(v_perplexity)\n",
    "          \n",
    "      decay_learning_rate(session, v_perplexity)\n",
    "        \n",
    "      # Generating new text ...\n",
    "      # We will be generating one segment having 500 bigrams\n",
    "      # Feel free to generate several segments by changing\n",
    "      # the value of segments_to_generate\n",
    "      print('Generated Text after epoch %d ... '%step)  \n",
    "      segments_to_generate = 1\n",
    "      chars_in_segment = 500\n",
    "    \n",
    "      for _ in range(segments_to_generate):\n",
    "        print('======================== New text Segment ==========================')\n",
    "        \n",
    "        # Start with a random word\n",
    "        test_word = np.zeros((1,vocabulary_size),dtype=np.float32)\n",
    "        rand_doc = data_list[np.random.randint(0,num_files)]\n",
    "        test_word[0,rand_doc[np.random.randint(0,len(rand_doc))]] = 1.0\n",
    "        print(\"\\t\",reverse_dictionary[np.argmax(test_word[0])],end='')\n",
    "        \n",
    "        # Generating words within a segment by feeding in the previous prediction\n",
    "        # as the current input in a recursive manner\n",
    "        for _ in range(chars_in_segment):    \n",
    "          sample_pred = session.run(test_prediction, feed_dict = {test_input:test_word})  \n",
    "          next_ind = sample(sample_pred.ravel())\n",
    "          test_word = np.zeros((1,vocabulary_size),dtype=np.float32)\n",
    "          test_word[0,next_ind] = 1.0\n",
    "          print(reverse_dictionary[next_ind],end='')\n",
    "        print(\"\")\n",
    "        \n",
    "        # Reset train state\n",
    "        session.run(reset_test_state)\n",
    "        print('====================================================================')\n",
    "      print(\"\")\n",
    "\n",
    "session.close()\n",
    "\n",
    "# Write training and validation perplexities to a csv file\n",
    "with open(filename_to_save, 'wt') as f:\n",
    "    writer = csv.writer(f, delimiter=',')\n",
    "    writer.writerow(train_perplexity_ot)\n",
    "    writer.writerow(valid_perplexity_ot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## LSTM with Beam-Search\n",
    "\n",
    "Here we alter the previously defined prediction related TensorFlow operations to employ beam-search. Beam search is a way of predicting several time steps ahead. Concretely instead of predicting the best prediction we have at a given time step, we get predictions for several time steps and get the sequence of highest joint probability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "beam_length = 5 # number of steps to look ahead\n",
    "beam_neighbors = 5 # number of neighbors to compare to at each step\n",
    "\n",
    "# We redefine the sample generation with beam search\n",
    "sample_beam_inputs = [tf.placeholder(tf.float32, shape=[1, vocabulary_size]) for _ in range(beam_neighbors)]\n",
    "\n",
    "best_beam_index = tf.placeholder(shape=None, dtype=tf.int32)\n",
    "best_neighbor_beam_indices = tf.placeholder(shape=[beam_neighbors], dtype=tf.int32)\n",
    "\n",
    "# Maintains output of each beam\n",
    "saved_sample_beam_output = [tf.Variable(tf.zeros([1, num_nodes])) for _ in range(beam_neighbors)]\n",
    "# Maintains the state of each beam\n",
    "saved_sample_beam_state = [tf.Variable(tf.zeros([1, num_nodes])) for _ in range(beam_neighbors)]\n",
    "\n",
    "# Resetting the sample beam states (should be done at the beginning of each text snippet generation)\n",
    "reset_sample_beam_state = tf.group(\n",
    "    *[saved_sample_beam_output[vi].assign(tf.zeros([1, num_nodes])) for vi in range(beam_neighbors)],\n",
    "    *[saved_sample_beam_state[vi].assign(tf.zeros([1, num_nodes])) for vi in range(beam_neighbors)]\n",
    ")\n",
    "\n",
    "# We stack them to perform gather operation below\n",
    "stacked_beam_outputs = tf.stack(saved_sample_beam_output)\n",
    "stacked_beam_states = tf.stack(saved_sample_beam_state)\n",
    "\n",
    "# The beam states for each beam (there are beam_neighbor-many beams) needs to be updated at every depth of tree\n",
    "# Consider an example where you have 3 classes where we get the best two neighbors (marked with star)\n",
    "#     a`      b*       c  \n",
    "#   / | \\   / | \\    / | \\\n",
    "#  a  b c  a* b` c  a  b  c\n",
    "# Since both the candidates from level 2 comes from the parent b\n",
    "# We need to update both states/outputs from saved_sample_beam_state/output to have index 1 (corresponding to parent b)\n",
    "update_sample_beam_state = tf.group(\n",
    "    *[saved_sample_beam_output[vi].assign(tf.gather_nd(stacked_beam_outputs,[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)],\n",
    "    *[saved_sample_beam_state[vi].assign(tf.gather_nd(stacked_beam_states,[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)]\n",
    ")\n",
    "\n",
    "# We calculate lstm_cell state and output for each beam\n",
    "sample_beam_outputs, sample_beam_states = [],[] \n",
    "for vi in range(beam_neighbors):\n",
    "    tmp_output, tmp_state = lstm_cell(\n",
    "        sample_beam_inputs[vi], saved_sample_beam_output[vi], saved_sample_beam_state[vi]\n",
    "    )\n",
    "    sample_beam_outputs.append(tmp_output)\n",
    "    sample_beam_states.append(tmp_state)\n",
    "\n",
    "# For a given set of beams, outputs a list of prediction vectors of size beam_neighbors\n",
    "# each beam having the predictions for full vocabulary\n",
    "sample_beam_predictions = []\n",
    "for vi in range(beam_neighbors):\n",
    "    with tf.control_dependencies([saved_sample_beam_output[vi].assign(sample_beam_outputs[vi]),\n",
    "                                saved_sample_beam_state[vi].assign(sample_beam_states[vi])]):\n",
    "        sample_beam_predictions.append(tf.nn.softmax(tf.nn.xw_plus_b(sample_beam_outputs[vi], w, b)))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the LSTM with Beam Search to Generate Text\n",
    "\n",
    "Here we train the LSTM on the available data and generate text using the trained LSTM for several steps. From each document we extract text for `steps_per_document` steps to train the LSTM on. We also report the train perplexity at the end of each step. Finally we test the LSTM by asking it to generate some new text with beam search starting from a randomly picked bigram."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning rate Decay Logic\n",
    "\n",
    "Here we define the logic to decrease learning rate whenever the validation perplexity does not decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning rate decay related\n",
    "# If valid perpelxity does not decrease\n",
    "# continuously for this many epochs\n",
    "# decrease the learning rate\n",
    "decay_threshold = 5\n",
    "# Keep counting perplexity increases\n",
    "decay_count = 0\n",
    "\n",
    "min_perplexity = 1e10\n",
    "\n",
    "# Learning rate decay logic\n",
    "def decay_learning_rate(session, v_perplexity):\n",
    "  global decay_threshold, decay_count, min_perplexity  \n",
    "  # Decay learning rate\n",
    "  if v_perplexity < min_perplexity:\n",
    "    decay_count = 0\n",
    "    min_perplexity= v_perplexity\n",
    "  else:\n",
    "    decay_count += 1\n",
    "\n",
    "  if decay_count >= decay_threshold:\n",
    "    print('\\t Reducing learning rate')\n",
    "    decay_count = 0\n",
    "    session.run(inc_gstep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the Beam Prediction Logic\n",
    "Here we define function that takes in the session as an argument and output a beam of predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test_sequences' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-6f872041e6bf>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    135\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mtest_sequences\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbest_beam_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    136\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 137\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest_sequences\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbest_beam_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m''\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'test_sequences' is not defined"
     ]
    }
   ],
   "source": [
    "test_word = None\n",
    "\n",
    "def get_beam_prediction(session):\n",
    "    \n",
    "    # Generating words within a segment with Beam Search\n",
    "    # To make some calculations clearer, we use the example as follows\n",
    "    # We have three classes with beam_neighbors=2 (best candidate denoted by *, second best candidate denoted by `)\n",
    "    # For simplicity we assume best candidate always have probability of 0.5 in output prediction\n",
    "    # second best has 0.2 output prediction\n",
    "    #           a`                   b*                   c                <--- root level\n",
    "    #    /     |     \\         /     |     \\        /     |     \\   \n",
    "    #   a      b      c       a*     b`     c      a      b      c         <--- depth 1\n",
    "    # / | \\  / | \\  / | \\   / | \\  / | \\  / | \\  / | \\  / | \\  / | \\\n",
    "    # a b c  a b c  a b c   a*b c  a`b c  a b c  a b c  a b c  a b c       <--- depth 2\n",
    "    # So the best beams at depth 2 would be\n",
    "    # b-a-a and b-b-a\n",
    "        \n",
    "    global test_word\n",
    "    global sample_beam_predictions\n",
    "    global update_sample_beam_state\n",
    "    \n",
    "    # Calculate the candidates at the root level\n",
    "    feed_dict = {}\n",
    "    for b_n_i in range(beam_neighbors):\n",
    "        feed_dict.update({sample_beam_inputs[b_n_i]: test_word})\n",
    "\n",
    "    # We calculate sample predictions for all neighbors with the same starting word/character\n",
    "    # This is important to update the state for all instances of beam search\n",
    "    sample_preds_root = session.run(sample_beam_predictions, feed_dict = feed_dict)  \n",
    "    sample_preds_root = sample_preds_root[0]\n",
    "\n",
    "    # indices of top-k candidates\n",
    "    # b and a in our example (root level)\n",
    "    this_level_candidates =  (np.argsort(sample_preds_root,axis=1).ravel()[::-1])[:beam_neighbors].tolist() \n",
    "\n",
    "    # probabilities of top-k candidates\n",
    "    # 0.5 and 0.2\n",
    "    this_level_probs = sample_preds_root[0,this_level_candidates] \n",
    "\n",
    "    # Update test sequence produced by each beam from the root level calculation\n",
    "    # Test sequence looks like for our example (at root)\n",
    "    # [b,a]\n",
    "    test_sequences = ['' for _ in range(beam_neighbors)]\n",
    "    for b_n_i in range(beam_neighbors):\n",
    "        test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]]\n",
    "\n",
    "    # Make the calculations for the rest of the depth of the beam search tree\n",
    "    for b_i in range(beam_length-1):\n",
    "        test_words = [] # candidate words for each beam\n",
    "        pred_words = [] # Predicted words of each beam\n",
    "\n",
    "        # computing feed_dict for the beam search (except root)\n",
    "        # feed dict should contain the best words/chars/bigrams found by the previous level of search\n",
    "\n",
    "        # For level 1 in our example this would be\n",
    "        # sample_beam_inputs[0]: b, sample_beam_inputs[1]:a\n",
    "        feed_dict = {}\n",
    "        for p_idx, pred_i in enumerate(this_level_candidates):                    \n",
    "            # Updating the feed_dict for getting next predictions\n",
    "            test_words.append(np.zeros((1,vocabulary_size),dtype=np.float32))\n",
    "            test_words[p_idx][0,this_level_candidates[p_idx]] = 1.0\n",
    "\n",
    "            feed_dict.update({sample_beam_inputs[p_idx]:test_words[p_idx]})\n",
    "\n",
    "        # Calculating predictions for all neighbors in beams\n",
    "        # This is a list of vectors where each vector is the prediction vector for a certain beam\n",
    "        # For level 1 in our example, the prediction values for \n",
    "        #      b             a  (previous beam search results)\n",
    "        # [a,  b,  c],  [a,  b,  c] (current level predictions) would be\n",
    "        # [0.1,0.1,0.1],[0.5,0.2,0]\n",
    "        sample_preds_all_neighbors = session.run(sample_beam_predictions, feed_dict=feed_dict)\n",
    "\n",
    "        # Create a single vector with \n",
    "        # Making our example [0.1,0.1,0.1,0.5,0.2,0] \n",
    "        sample_preds_all_neighbors_concat = np.concatenate(sample_preds_all_neighbors,axis=1)\n",
    "\n",
    "        # Update this_level_candidates to be used for the next iteration\n",
    "        # And update the probabilities for each beam\n",
    "        # In our example these would be [3,4] (indices with maximum value from above vector)\n",
    "        this_level_candidates = np.argsort(sample_preds_all_neighbors_concat.ravel())[::-1][:beam_neighbors]\n",
    "\n",
    "        # In the example this would be [1,1]\n",
    "        parent_beam_indices = this_level_candidates//vocabulary_size\n",
    "\n",
    "        # normalize this_level_candidates to fall between [0,vocabulary_size]\n",
    "        # In this example this would be [0,1]\n",
    "        this_level_candidates = (this_level_candidates%vocabulary_size).tolist()\n",
    "\n",
    "        # Here we update the final state of each beam to be\n",
    "        # the state that was at the index 1. Because for both the candidates at this level the parent is \n",
    "        # at index 1 (that is b from root level)\n",
    "        session.run(update_sample_beam_state, feed_dict={best_neighbor_beam_indices: parent_beam_indices})\n",
    "\n",
    "        # Here we update the joint probabilities of each beam and add the newly found candidates to the sequence\n",
    "        tmp_this_level_probs = np.asarray(this_level_probs) #This is currently [0.5,0.2]\n",
    "        tmp_test_sequences = list(test_sequences) # This is currently [b,a]\n",
    "\n",
    "        for b_n_i in range(beam_neighbors):\n",
    "            # We make the b_n_i element of this_level_probs to be the probability of parents\n",
    "            # In the example the parent indices are [1,1]\n",
    "            # So this_level_probs become [0.5,0.5]\n",
    "            this_level_probs[b_n_i] = tmp_this_level_probs[parent_beam_indices[b_n_i]]\n",
    "\n",
    "            # Next we multipyle these by the probabilities of the best candidates from current level \n",
    "            # [0.5*0.5, 0.5*0.2] = [0.25,0.1]\n",
    "            this_level_probs[b_n_i] *= sample_preds_all_neighbors[parent_beam_indices[b_n_i]][0,this_level_candidates[b_n_i]]\n",
    "\n",
    "            # Make the b_n_i element of test_sequences to be the correct parent of the current best candidates\n",
    "            # In the example this becomes [b, b]\n",
    "            test_sequences[b_n_i] = tmp_test_sequences[parent_beam_indices[b_n_i]]\n",
    "\n",
    "            # Now we append the current best candidates\n",
    "            # In this example this becomes [ba,bb]\n",
    "            test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]]\n",
    "\n",
    "            # Create one-hot-encoded representation for each candidate\n",
    "            pred_words.append(np.zeros((1,vocabulary_size),dtype=np.float32))\n",
    "            pred_words[b_n_i][0,this_level_candidates[b_n_i]] = 1.0\n",
    "\n",
    "    # Calculate best beam id based on the highest beam probability\n",
    "    # Using the highest beam probability always lead to very monotonic text\n",
    "    # Let us sample one randomly where one being sampled is decided by the likelihood of that beam\n",
    "    rand_cand_ids = np.argsort(this_level_probs)[-3:]\n",
    "    rand_cand_probs = this_level_probs[rand_cand_ids]/np.sum(this_level_probs[rand_cand_ids])\n",
    "    random_id = np.random.choice(rand_cand_ids,p=rand_cand_probs)\n",
    "\n",
    "    best_beam_id = parent_beam_indices[random_id]\n",
    "\n",
    "    # Update state and output variables for test prediction\n",
    "    session.run(update_sample_beam_state,feed_dict={best_neighbor_beam_indices:[best_beam_id for _ in range(beam_neighbors)]})\n",
    "\n",
    "    # Make the last word/character/bigram from the best beam\n",
    "    test_word = pred_words[best_beam_id]\n",
    "    \n",
    "    return test_sequences[best_beam_id]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running Training, Validation and Generation\n",
    "\n",
    "We traing the LSTM on existing training data, check the validaiton perplexity on an unseen chunk of text and generate a fresh segment of text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "(40).(71).(52).(81).(84).(8).(13).(19).(63).(28).\n",
      "Average loss at step 1: 4.428126\n",
      "\tPerplexity at step 1: 83.774273\n",
      "\n",
      "Valid Perplexity: 74.72\n",
      "\n",
      "Generated Text after epoch 0 ... \n",
      "======================== New text Segment ==========================\n",
      " .  with the laught to laught and in the wang the chas the was laugh, and but ther hang the changeling to to her hang ther hang, and took to they her, and, anythe she was that he was to to her was the wang, who\n",
      "hen her was to to the was the wang to laugh, and but her, who\n",
      "child, she her, who\n",
      "child, she begand he her, whicen to the to the cames, and he her, who\n",
      "child, and her, when her, when her, whic her, whice, and in laugh, and with the would, anyegan the begand her was the wang, when ther hat they her, and, anythe to the cames, and he her, who\n",
      "child, and her, when her, when her, whic her, whice, and in laugh, and but ther hang the cegand her was the wang, who\n",
      "hen the was took her, but the wang to her was the wang, when ther hangel and to he her, and took to they her, and, anythe to the cames, and he her, who\n",
      "child, anyegan the beang, and took to ther her, and, anyone she was ther, when ther hat that her, and, anythe to the cames, and he her, who\n",
      "child, anyegan the chang, whic her, whic\n",
      "====================================================================\n",
      "\n",
      "(63).(37).(98).(79).(71).(64).(25).(84).(34).(41).\n",
      "Average loss at step 2: 3.205511\n",
      "\tPerplexity at step 2: 24.668109\n",
      "\n",
      "Valid Perplexity: 41.15\n",
      "\n",
      "Generated Text after epoch 1 ... \n",
      "======================== New text Segment ==========================\n",
      " ather, and said,\n",
      "     when she will, and they went out of the were the well,\n",
      "     when she window, and and the well,\n",
      "     when she window, and and the well, and then the fairest of all.\n",
      "\n",
      "then when she with of the houself and they wept and was her, and snow-white was she was said, at on the well,\n",
      "               now-white as and she said, and was she was said, and said what had she said, and look of thought, where warfs, and said when she window, and was she was and they went, and, and was they went, and they had and taid, and said,\n",
      "      and was said,\n",
      "     when she coffin, and the wast when she said, and thought, and, and to lass thall, and the with the window, and and the well,\n",
      "     when she window, and and the well,\n",
      "     when she window, and the was again they went, and with the well,\n",
      "             when she was her, and snow-white she was said, and that the fairest of all.\n",
      "\n",
      "then warfs, alled to the withe was of the cout of the dould, and when she said, and was said what had she had said,\n",
      "====================================================================\n",
      "\n",
      "(77).(82).(26).(12).(91).(31).(39).(71).(90).(56).\n",
      "Average loss at step 3: 2.369584\n",
      "\tPerplexity at step 3: 10.692940\n",
      "\n",
      "Valid Perplexity: 56.13\n",
      "\n",
      "Generated Text after epoch 2 ... \n",
      "======================== New text Segment ==========================\n",
      " that the\n",
      "foxed to her, farewell, dear mrs.s. gossip, i the roaching you\n",
      "have had dear mrs. glaughed heartily at her, and bounded off at the periants, lay safe and sound to her house.then the\n",
      "fox cried that her, full, dear s. gossip, i the roasting you\n",
      "herself\n",
      "outside.  the\n",
      "laughed lay heartily at her, so her, and bounded, and dragged herself\n",
      "outside.\n",
      "\n",
      "there lay the fox, who pretented to be roasting all, dear s. gossip, may the roasting you\n",
      "have had do you good, laughed heartily at her, at her, and bounded off the roaew i have you good, laughed herself was ly able to walk slowed, but she was in such\n",
      "concern\n",
      "about the fox that she took him on her back, and the slowly\n",
      "carried him who was perfectly safe and sound to her house.  then the pox cried\n",
      "to her, farewell, dear mrs. gossip, may.  the roasting you\n",
      "have had do you good, laughed itheartily be her, asked and burred him who was peperfectly was only a from and perish, you.  then she complaints,\n",
      "and slowly\n",
      "carried him who was perfrftly safe\n",
      "====================================================================\n",
      "\n",
      "(73).(84).(92).(86).(9).(28).(90)."
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-21-7164b0af6634>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     65\u001b[0m             \u001b[1;31m# Running the TensorFlow operations\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m             _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n\u001b[1;32m---> 67\u001b[1;33m                                                        feed_dict=feed_dict)\n\u001b[0m\u001b[0;32m     68\u001b[0m             \u001b[1;31m# Update doc_perpelxity variable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     69\u001b[0m             \u001b[0mdoc_perplexity\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mstep_perplexity\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    903\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 905\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    906\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1135\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1137\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1138\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1353\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1355\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1356\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1357\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1359\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1360\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1361\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1362\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1338\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[1;32m-> 1340\u001b[1;33m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[0;32m   1341\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1342\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "filename_to_save = 'lstm_beam_search_dropout'\n",
    "\n",
    "# Some hyperparameters needed for the training process\n",
    "\n",
    "num_steps = 26\n",
    "steps_per_document = 100\n",
    "valid_summary = 1\n",
    "train_doc_count = 100\n",
    "docs_per_step = 10\n",
    "\n",
    "\n",
    "beam_nodes = []\n",
    "\n",
    "beam_train_perplexity_ot = []\n",
    "beam_valid_perplexity_ot = []\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "print('Initialized')\n",
    "average_loss = 0\n",
    "\n",
    "# We use the first 10 documents that has \n",
    "# more than 10*steps_per_document bigrams for creating the validation dataset\n",
    "\n",
    "# Identify the first 10 documents following the above condition\n",
    "long_doc_ids = []\n",
    "for di in range(num_files):\n",
    "  if len(data_list[di])>10*steps_per_document:\n",
    "    long_doc_ids.append(di)\n",
    "  if len(long_doc_ids)==10:\n",
    "    break\n",
    "    \n",
    "# Generating validation data\n",
    "data_gens = []\n",
    "valid_gens = []\n",
    "for fi in range(num_files):\n",
    "  # Get all the bigrams if the document id is not in the validation document ids\n",
    "  if fi not in long_doc_ids:\n",
    "    data_gens.append(DataGeneratorOHE(data_list[fi],batch_size,num_unrollings))\n",
    "  # if the document is in the validation doc ids, only get up to the \n",
    "  # last steps_per_document bigrams and use the last steps_per_document bigrams as validation data\n",
    "  else:\n",
    "    data_gens.append(DataGeneratorOHE(data_list[fi][:-steps_per_document],batch_size,num_unrollings))\n",
    "    # Defining the validation data generator\n",
    "    valid_gens.append(DataGeneratorOHE(data_list[fi][-steps_per_document:],1,1))\n",
    "\n",
    "\n",
    "feed_dict = {}\n",
    "for step in range(num_steps):\n",
    "    \n",
    "    for di in np.random.permutation(train_doc_count)[:docs_per_step]:            \n",
    "        doc_perplexity = 0\n",
    "        for doc_step_id in range(steps_per_document):\n",
    "            \n",
    "            # Get a set of unrolled batches\n",
    "            u_data, u_labels = data_gens[di].unroll_batches()\n",
    "            \n",
    "            # Populate the feed dict by using each of the data batches\n",
    "            # present in the unrolled data\n",
    "            for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):            \n",
    "                feed_dict[train_inputs[ui]] = dat\n",
    "                feed_dict[train_labels[ui]] = lbl\n",
    "            \n",
    "            # Running the TensorFlow operations\n",
    "            _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n",
    "                                                       feed_dict=feed_dict)\n",
    "            # Update doc_perpelxity variable\n",
    "            doc_perplexity += step_perplexity \n",
    "            \n",
    "            # Update the average_loss variable\n",
    "            average_loss += step_perplexity \n",
    "            \n",
    "        # Show the printing progress <train_doc_id_1>.<train_doc_id_2>. ...\n",
    "        print('(%d).'%di,end='') \n",
    "    \n",
    "    # resetting hidden state after processing a single document\n",
    "    # It's still questionable if this adds value in terms of learning\n",
    "    # One one hand it's intuitive to reset the state when learning a new document\n",
    "    # On the other hand this approach creates a bias for the state to be zero\n",
    "    # We encourage the reader to investigate further the effect of resetting the state\n",
    "    #session.run(reset_train_state) # resetting hidden state for each document\n",
    "    print('')\n",
    "    \n",
    "    if (step+1) % valid_summary == 0:\n",
    "      \n",
    "      # Compute average loss\n",
    "      average_loss = average_loss / (docs_per_step*steps_per_document*valid_summary)\n",
    "      \n",
    "      # Print loss\n",
    "      print('Average loss at step %d: %f' % (step+1, average_loss))\n",
    "      print('\\tPerplexity at step %d: %f' %(step+1, np.exp(average_loss)))\n",
    "      beam_train_perplexity_ot.append(np.exp(average_loss))\n",
    "    \n",
    "      average_loss = 0 # reset loss\n",
    "        \n",
    "      valid_loss = 0 # reset loss\n",
    "        \n",
    "      # calculate valid perplexity\n",
    "      for v_doc_id in range(10):\n",
    "          # Remember we process things as bigrams\n",
    "          # So need to divide by 2\n",
    "          for v_step in range(steps_per_document//2):\n",
    "            uvalid_data,uvalid_labels = valid_gens[v_doc_id].unroll_batches()        \n",
    "\n",
    "            # Run validation phase related TensorFlow operations       \n",
    "            v_perp = session.run(\n",
    "                valid_perplexity_without_exp,\n",
    "                feed_dict = {valid_inputs:uvalid_data[0],valid_labels: uvalid_labels[0]}\n",
    "            )\n",
    "\n",
    "            valid_loss += v_perp\n",
    "            \n",
    "          session.run(reset_valid_state)\n",
    "      \n",
    "          # Reset validation data generator cursor\n",
    "          valid_gens[v_doc_id].reset_indices()      \n",
    "    \n",
    "      print()\n",
    "      v_perplexity = np.exp(valid_loss/(steps_per_document*10.0//2))\n",
    "      print(\"Valid Perplexity: %.2f\\n\"%v_perplexity)\n",
    "      beam_valid_perplexity_ot.append(v_perplexity)\n",
    "      \n",
    "      # Decay learning rate\n",
    "      decay_learning_rate(session, v_perplexity)\n",
    "    \n",
    "      # Generating new text ...\n",
    "      # We will be generating one segment having 500 bigrams\n",
    "      # Feel free to generate several segments by changing\n",
    "      # the value of segments_to_generate\n",
    "      print('Generated Text after epoch %d ... '%step)  \n",
    "      segments_to_generate = 1\n",
    "      chars_in_segment = 500//beam_length\n",
    "    \n",
    "      for _ in range(segments_to_generate):\n",
    "        print('======================== New text Segment ==========================')\n",
    "        # first word randomly generated\n",
    "        test_word = np.zeros((1,vocabulary_size),dtype=np.float32)\n",
    "        rand_doc = data_list[np.random.randint(0,num_files)]\n",
    "        test_word[0,rand_doc[np.random.randint(0,len(rand_doc))]] = 1.0\n",
    "        print(\"\",reverse_dictionary[np.argmax(test_word[0])],end='')\n",
    "        \n",
    "        for _ in range(chars_in_segment):\n",
    "            \n",
    "            test_sequence = get_beam_prediction(session)\n",
    "            print(test_sequence,end='')\n",
    "            \n",
    "        print(\"\")\n",
    "        session.run([reset_sample_beam_state])\n",
    "        \n",
    "        print('====================================================================')\n",
    "      print(\"\")\n",
    "\n",
    "session.close()\n",
    "    \n",
    "with open(filename_to_save, 'wt') as f:\n",
    "    writer = csv.writer(f, delimiter=',')\n",
    "    writer.writerow(beam_train_perplexity_ot)\n",
    "    writer.writerow(beam_valid_perplexity_ot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
